ibm-watsonx-gov 1.3.3__cp313-cp313-win_amd64.whl

ibm_watsonx_gov/providers/llmevalkit/function_calling/metrics/parameter/parameter_metrics_runtime.json

@@ -0,0 +1,489 @@
+[
+    {
+        "name": "parameter_hallucination_check",
+        "task_description": "You are an expert evaluator assessing whether a **specific parameter value**-identified as `parameter_name`-in a tool call is **grounded** in the provided conversation history or tool specification, or whether it is **hallucinated**.\n\nYour task is to make a **strictly evidence-based** judgment. Evaluate this parameter alone-ignore other parameters or external knowledge. The value must:\n- Be explicitly supported by the dialogue, prior tool calls, or tool specification\n- Be appropriate for the context and user intent\n- Respect constraint rules defined in the specification\n\nIMPORTANT: Evaluate ONLY the **specific parameter value** provided. Do not evaluate the entire tool call correctness, other parameters, or the function itself. Focus solely on whether this parameter value is grounded in the conversation or specification.\n---\n\n### Rating Scale\n\nAssign a score from 1 to 5 based on how clearly the value is grounded:\n\n**5 - Perfectly Grounded**  \nExplicitly stated in conversation or matches a clearly documented and contextually appropriate default.\n\n**4 - Mostly Grounded**  \nNot directly quoted but clearly follows from the conversation or is a minor variation on a documented default.\n\n**3 - Ambiguous**  \nRequires non-trivial inference or transformation to relate the value to the dialogue or specification.\n\n**2 - Mostly Ungrounded**  \nLoosely related, incorrectly formatted, or partially contradicts the spec or context.\n\n**1 - Completely Ungrounded**  \nInvented, unrelated, or clearly in conflict with the dialogue or specification.\n\nBe conservative in your scoring: Use 1-3 for weak or unverified grounding, and 4-5 only when evidence is explicit or clearly reliable.\n\n---\n\n### Acceptable Sources for Grounding\nA value may be considered grounded if it comes from:\n- Explicit user input\n- Clearly implied user intent (minimal inference)\n- Assistant statements explicitly confirmed by the user\n- Outputs of previous tool calls\n- Documented default values in the tool specification\n\n---\n\n### Ungrounded Patterns\nMark a value ungrounded if:\n- It is absent from both conversation and tool spec\n- It depends on vague or missing input (e.g., \"tomorrow\" without a date format)\n- It uses the wrong type, unit, or format\n- It contradicts other parameters or tool constraints\n- It inaccurately rephrases or transforms the user's input\n\n---\n\n### Handling Defaults\nA default value may be accepted only if:\n- It is explicitly documented in the tool spec\n- It fits the context of the user's request\n- It does not contradict stated or implied intent\nOtherwise, treat the value as ungrounded.\n\n---\n\n### Parameter Relationships\nEnsure this parameter does not contradict other parameters in the same call.\nExample: If `add_day=false` and `day=\"Sunday\"` but the spec says `add_day` must be true to use `day`, the `day` value is ungrounded.\n\n---\n\n### Final Guideline\nLarge language models frequently hallucinate parameter values or apply defaults inappropriately. Your careful and conservative evaluation prevents propagation of such errors.",
+        "jsonschema": {
+            "title": "parameter_hallucination_check",
+            "description": "Assessment of tool call parameter hallucination, following the rubric defined above.",
+            "type": "object",
+            "additionalProperties": false,
+            "properties": {
+                "evidence": {
+                    "type": "string",
+                    "description": "Provide a short sentence or two with EXACT quotes from the conversation history (user messages, assistant responses, previous tool outputs) or tool specification that directly support your assessment. For grounded values, cite the specific text where this value originates. For ungrounded values, demonstrate the absence of supporting evidence or cite contradictory information."
+                },
+                "explanation": {
+                    "type": "string",
+                    "description": "Explain in one or two sentences why the specific parameter value is grounded or ungrounded. Reference exact sources from the conversation or tool specification that either support or contradict the parameter value. Address: 1) Where the value originated from, 2) Whether this source is sufficient for grounding, 3) Any format or type compliance issues, 4) Any contradictions with other parameters or specifications, and 5) For default values, whether they're documented and appropriate for this context."
+                },
+                "output": {
+                    "type": "integer",
+                    "minimum": 1,
+                    "maximum": 5,
+                    "threshold_low": 4,
+                    "threshold_high": 5,
+                    "description": "Parameter Hallucination Score (1-5):\n\n- 5: Perfectly grounded\n  The parameter value is directly quoted in the conversation or exactly matches a documented default that fits the users intent.\n  Example:\n  User: \"Book a hotel in Rome.\"\n  → Tool call: search_hotels(city=\"Rome\")\n\n- 4: Grounded via logical reasoning\n  The value is not directly quoted but clearly follows from the conversation through straightforward, logic-based inference.\n  Example:\n  User: \"I'd like a room for me and my wife.\"\n  → Tool call: search_hotels(guests=2)\n\n- 3: Ambiguous\n  The value might relate to the conversation or domain context, but grounding requires assumptions or nontrivial interpretation not clearly supported by evidence.\n  Example:\n  User: \"Let's go somewhere warm.\"\n  → Tool call: search_hotels(city=\"Barcelona\")\n\n- 2: Likely hallucinated\n  The value is loosely connected to the topic but lacks grounding in the conversation or tool specification, or only partially aligns.\n  Example:\n  User: \"Find hotels.\"\n  → Tool call: search_hotels(city=\"Berlin\")\n  (No city mentioned)\n\n- 1: Clearly hallucinated\n  The parameter value is invented, irrelevant, or contradicts the dialogue or tool specification.\n  Example:\n  User: \"Book a hotel.\"\n  → Tool call: search_hotels(city=\"Mars\")"
+                },
+                "confidence": {
+                    "type": "number",
+                    "minimum": 0,
+                    "maximum": 1,
+                    "threshold_low": 0,
+                    "threshold_high": 1,
+                    "description": "Confidence in the accuracy of this assessment (range: 0.0-1.0). Assign higher confidence when evidence is clear, complete, and consistent, and lower confidence when it is ambiguous, incomplete, or conflicting."
+                },
+                "correction": {
+                    "type": "object",
+                    "properties": {
+                        "reason_types": {
+                            "type": "array",
+                            "description": "A short sentence or two with types of issues with the parameter value, if any. Use one or more of these values: FORMAT_ERROR (wrong format or type), MISSING_INFORMATION (needs more data), PARAMETER_CONTRADICTION (conflicts with other parameters), DEFAULT_ISSUE (inappropriate default), OTHER (explain in reasons).",
+                            "items": {
+                                "type": "string",
+                                "enum": [
+                                    "FORMAT_ERROR",
+                                    "MISSING_INFORMATION",
+                                    "PARAMETER_CONTRADICTION",
+                                    "DEFAULT_ISSUE",
+                                    "OTHER"
+                                ]
+                            }
+                        },
+                        "reasons": {
+                            "type": "string",
+                            "description": "Concise explanation of the specific issues with the parameter value."
+                        },
+                        "parameter": {
+                            "type": "object",
+                            "description": "Object containing parameter name as key and corrected value as value. Use 'need_more_information' as the key and clarification question for the user if additional information is required. Use 'need_more_tool_calls' if the value cannot be corrected without additional tool calls. If the value is well-grounded, return an empty object {}.",
+                            "additionalProperties": true
+                        }
+                    },
+                    "description": "For well-grounded values: Provide an empty object {}. For ungrounded values: Provide an object with reason_types, reasons, and parameter correction.",
+                    "required": []
+                }
+            },
+            "required": [
+                "explanation",
+                "evidence",
+                "output",
+                "confidence",
+                "correction"
+            ]
+        },
+        "examples": [
+            {
+                "user_kwargs": {
+                    "conversation_context": [
+                        {
+                            "role": "user",
+                            "content": "Translate 'hello' to Spanish."
+                        }
+                    ],
+                    "tool_inventory": [
+                        {
+                            "type": "function",
+                            "function": {
+                                "name": "translate_text",
+                                "description": "Translate given text",
+                                "parameters": {
+                                    "type": "object",
+                                    "properties": {
+                                        "text": {
+                                            "type": "string",
+                                            "description": "The text to be translated"
+                                        },
+                                        "target_lang": {
+                                            "type": "string",
+                                            "description": "The target language code using ISO 639-1 (e.g., 'en' for English, 'es' for Spanish, 'fr' for French)"
+                                        }
+                                    },
+                                    "required": [
+                                        "text",
+                                        "target_lang"
+                                    ]
+                                }
+                            }
+                        }
+                    ],
+                    "tool_call": {
+                        "id": "call_001",
+                        "type": "function",
+                        "function": {
+                            "name": "translate_text",
+                            "arguments": "{ \"text\": \"hello\", \"target_lang\": \"es\" }"
+                        }
+                    },
+                    "parameter_name": "target_lang",
+                    "parameter_value": "es"
+                },
+                "output": {
+                    "evidence": "User said: \"Translate 'hello' to Spanish.\" The tool spec defines target_lang as a string. 'es' is the ISO 639-1 code for Spanish.",
+                    "explanation": "The value 'es' is a grounded and appropriate transformation of the user's explicit request for Spanish. Assuming standard practice for language codes in translation APIs, the ISO code 'es' matches both the user intent and specification.",
+                    "output": 5,
+                    "confidence": 0.97,
+                    "correction": {}
+                }
+            },
+            {
+                "user_kwargs": {
+                    "conversation_context": [
+                        {
+                            "role": "user",
+                            "content": "Fetch my latest tweets."
+                        }
+                    ],
+                    "tool_inventory": [
+                        {
+                            "type": "function",
+                            "function": {
+                                "name": "get_tweets",
+                                "description": "Retrieve recent tweets",
+                                "parameters": {
+                                    "type": "object",
+                                    "properties": {
+                                        "username": {
+                                            "type": "string",
+                                            "description": "The Twitter username to fetch tweets for"
+                                        },
+                                        "count": {
+                                            "type": "integer",
+                                            "description": "The number of recent tweets to retrieve"
+                                        }
+                                    },
+                                    "required": [
+                                        "username",
+                                        "count"
+                                    ]
+                                }
+                            }
+                        }
+                    ],
+                    "tool_call": {
+                        "id": "call_001",
+                        "type": "function",
+                        "function": {
+                            "name": "get_tweets",
+                            "arguments": "{ \"username\": \"elonmusk\", \"count\": 20 }"
+                        }
+                    },
+                    "parameter_name": "count",
+                    "parameter_value": 20
+                },
+                "output": {
+                    "evidence": "User said: \"Fetch my latest tweets.\" There is no mention of a tweet count. Tool spec lists 'count' as an integer parameter but provides no default.",
+                    "explanation": "The value 20 is ungrounded. The user did not specify how many tweets to fetch, and the tool specification does not define a default value for the 'count' parameter. The agent's choice appears arbitrary.",
+                    "output": 1,
+                    "confidence": 0.95,
+                    "correction": {
+                        "reason_types": [
+                            "MISSING_INFORMATION"
+                        ],
+                        "reasons": "The value was invented without support in the user message or spec. There is no evidence for selecting 20.",
+                        "parameter": {
+                            "need_more_information": "How many tweets would you like to retrieve? Please specify a number."
+                        }
+                    }
+                }
+            },
+            {
+                "user_kwargs": {
+                    "conversation_context": [
+                        {
+                            "role": "user",
+                            "content": "Can you show me just a few of my latest messages?"
+                        }
+                    ],
+                    "tool_inventory": [
+                        {
+                            "type": "function",
+                            "function": {
+                                "name": "get_messages",
+                                "description": "Retrieve user's messages",
+                                "parameters": {
+                                    "type": "object",
+                                    "properties": {
+                                        "user_id": {
+                                            "type": "integer",
+                                            "description": "The ID of the user whose messages are being retrieved"
+                                        },
+                                        "limit": {
+                                            "type": "integer",
+                                            "description": "The maximum number of messages to retrieve"
+                                        }
+                                    },
+                                    "required": [
+                                        "user_id",
+                                        "limit"
+                                    ]
+                                }
+                            }
+                        }
+                    ],
+                    "tool_call": {
+                        "id": "call_002",
+                        "type": "function",
+                        "function": {
+                            "name": "get_messages",
+                            "arguments": "{ \"user_id\": 456, \"limit\": 3 }"
+                        }
+                    },
+                    "parameter_name": "limit",
+                    "parameter_value": 3
+                },
+                "output": {
+                    "evidence": "User said: 'just a few of my latest messages.' The phrase 'a few' is vague and subjective, and while 3 is a reasonable interpretation, it is not explicitly stated.",
+                    "explanation": "The value limit=3 is a plausible interpretation of the user's request for 'a few' messages. However, the term is ambiguous and does not unambiguously support the specific value of 3. The assistant made a judgment call based on conversational context, but without explicit grounding or a documented default, the score is 3.",
+                    "output": 3,
+                    "confidence": 0.79,
+                    "correction": {
+                        "reason_types": [
+                            "MISSING_INFORMATION"
+                        ],
+                        "reasons": "The user's phrasing is vague and does not explicitly support the value 3. Clarification is needed to ensure the correct value is used.",
+                        "parameter": {
+                            "need_more_information": "You asked for 'a few' messages. Could you specify how many exactly you'd like to see?"
+                        }
+                    }
+                }
+            }
+        ]
+    },
+    {
+        "name": "parameter_value_format_alignment",
+        "task_description": "Evaluate whether the specific parameter value (named in \"parameter_name\") conforms exactly to the required type, format, and unit conventions defined in the API specification.\n\nYour judgment must be strictly evidence-based: rely exclusively on the provided API documentation and the parameter value. Do not assume or infer formatting rules that are not explicitly documented.\n\nIMPORTANT: Evaluate ONLY the **specific parameter value** provided to ensure it meets the exact type, format, and unit requirements. Do not evaluate the entire tool call correctness, other parameters, or the function itself. Focus solely on whether this parameter value aligns with the specification.\n---\n\n### 1. Data Type Compliance (Output = 4-5)\n- Check that the value matches the required data type (e.g., string, integer, boolean, object, array)\n- Verify that numbers are represented as numeric types, not strings\n- Ensure strings are quoted correctly if required\n- Confirm booleans are true/false (not \"true\"/\"false\") when the type is boolean\n\n---\n\n### 2. Format Specification Compliance\n- Validate adherence to documented format constraints, including:\n  - Date and time formats (e.g., ISO 8601, YYYY-MM-DD, timezone presence)\n  - Currency (e.g., $USD, symbol placement, decimal precision)\n  - Special patterns (e.g., phone numbers, email addresses, postal codes)\n- Enforce regular expressions or formatting rules defined in the schema\n\n---\n\n### 3. Unit Compliance\n- Verify presence or absence of unit suffixes/prefixes as specified (e.g., \"kg\", \"seconds\")\n- Confirm unit types are consistent with spec (e.g., Celsius vs Fahrenheit)\n- Reject any extra units when raw values are required\n\n---\n\n### 4. Consistency Requirements\n- Ensure values are consistent with related parameters (e.g., both timestamps in UTC)\n- Validate compatibility with other parameter values in the same call\n\n---\n\n### Format Alignment Rating Scale\n- **5 - Perfect Alignment**: Value exactly matches required type, format, and units\n- **4 - Minor Deviation**: Value is fundamentally correct but with minor formatting issues\n- **3 - Moderate Deviation**: Type or format is partially incorrect; may cause issues\n- **2 - Major Deviation**: Type and format are significantly wrong; likely to fail\n- **1 - Complete Mismatch**: Value is entirely incompatible with specification\n\n---\n\n### Final Rule\nBe conservative in your assessment:\n- If the spec is ambiguous and the value seems plausible, prefer a higher score (4-5)\n- If the spec is clear and the value deviates, assign a lower score (1-3)\n\nYour evaluation is critical: even minor formatting issues can lead to tool failures or incorrect outputs. Careful review ensures reliability in downstream tool use.",
+        "jsonschema": {
+            "title": "parameter_value_format_alignment",
+            "description": "Assessment of tool call parameter value format alignment, following the rubric defined above.",
+            "type": "object",
+            "additionalProperties": false,
+            "properties": {
+                "evidence": {
+                    "type": "string",
+                    "description": "Give a short sentence or two with quotes of the spec's type/format definition and the actual provided value that supports your judgment. Include EXACT text from the tool specification regarding this parameter's requirements."
+                },
+                "explanation": {
+                    "type": "string",
+                    "description": "Explain in one or two sentence precisely why the parameter value does or does not match the specification's type, format, and unit requirements. Reference specific details from the parameter specification including data type, format requirements, unit conventions, and any pattern constraints. For non-conforming values, explain exactly what is incorrect and how it deviates from requirements."
+                },
+                "output": {
+                    "type": "integer",
+                    "minimum": 1,
+                    "maximum": 5,
+                    "threshold_low": 4,
+                    "threshold_high": 5,
+                    "description": "An integer from 1 to 5 indicating how well the parameter value conforms to the required type, format, and unit as defined in the API specification.\n\nScore meanings:\n\n5 - Perfect Alignment: The value exactly matches the expected type, format, and unit. Example: expected date in 'YYYY-MM-DD', value is '2025-08-05'.\n\n4 - Minor Deviation: The value has small formatting issues (e.g., missing leading zeros) but is still likely to be accepted by the API or parser unless strict validation is enforced. Example: expected '2025-08-05', value is '2025-8-5'.\n\n3 - Moderate Deviation: The value partially matches the expected format but is likely to be rejected or misinterpreted by automatic processing. Example: expected 'YYYY-MM-DD', value is 'August 5, 2025'.\n\n2 - Major Deviation: The value significantly violates the expected type, format, or unit and is very likely to fail. Example: expected 'YYYY-MM-DD', value is '08/05/2025'.\n\n1 - Complete Mismatch: The value is entirely incompatible with the required format or type. Example: expected boolean, value is 'maybe'."
+                },
+                "confidence": {
+                    "type": "number",
+                    "minimum": 0,
+                    "maximum": 1,
+                    "threshold_low": 0,
+                    "threshold_high": 1,
+                    "description": "Confidence in the accuracy of this assessment (range: 0.0-1.0). Assign higher confidence when evidence is clear, complete, and consistent, and lower confidence when it is ambiguous, incomplete, or conflicting."
+                },
+                "correction": {
+                    "type": "object",
+                    "properties": {
+                        "reason_types": {
+                            "type": "array",
+                            "description": "A short sentence or two with types of format issues with the parameter value, if any. Use one or more of these values: TYPE_ERROR (wrong data type), FORMAT_ERROR (incorrect format), UNIT_ERROR (wrong/missing units), PATTERN_ERROR (doesn't match required pattern), CONSISTENCY_ERROR (inconsistent with related parameters), OTHER (explain in reasons).",
+                            "items": {
+                                "type": "string",
+                                "enum": [
+                                    "TYPE_ERROR",
+                                    "FORMAT_ERROR",
+                                    "UNIT_ERROR",
+                                    "PATTERN_ERROR",
+                                    "CONSISTENCY_ERROR",
+                                    "OTHER"
+                                ]
+                            }
+                        },
+                        "reasons": {
+                            "type": "string",
+                            "description": "Concise explanation of the specific format issues with the parameter value."
+                        },
+                        "parameter": {
+                            "type": "object",
+                            "description": "An object containing the corrected parameter value in the form: { \"<parameter_name>\": <corrected_value> }. If you cannot correct the value, use 'need_more_information' as the key and provide a clarification question for the user. Use 'need_more_tool_calls' if the value cannot be corrected without additional tool calls. If the value is well-formatted, return an empty object {}.",
+                            "additionalProperties": true
+                        }
+                    },
+                    "description": "For correctly formatted values: Provide an empty object {}. For format issues: Provide an object with reason_types, reasons, and parameter correction.",
+                    "required": []
+                }
+            },
+            "required": [
+                "explanation",
+                "evidence",
+                "output",
+                "confidence",
+                "correction"
+            ]
+        },
+        "examples": [
+            {
+                "user_kwargs": {
+                    "conversation_context": [
+                        {
+                            "role": "user",
+                            "content": "Set a short countdown for 12 minutes."
+                        }
+                    ],
+                    "tool_inventory": [
+                        {
+                            "type": "function",
+                            "function": {
+                                "name": "set_timer",
+                                "description": "Set a countdown timer",
+                                "parameters": {
+                                    "type": "object",
+                                    "properties": {
+                                        "duration": {
+                                            "type": "string",
+                                            "description": "Duration in the format 'number + unit' (e.g., '5 minutes', '30 seconds')"
+                                        }
+                                    },
+                                    "required": [
+                                        "duration"
+                                    ]
+                                }
+                            }
+                        }
+                    ],
+                    "tool_call": {
+                        "id": "call_001",
+                        "type": "function",
+                        "function": {
+                            "name": "set_timer",
+                            "arguments": "{ \"duration\": \"12 minutes\" }"
+                        }
+                    },
+                    "parameter_name": "duration",
+                    "parameter_value": "12 minutes"
+                },
+                "output": {
+                    "evidence": "The tool spec states that 'duration' must be a string in the format 'number + unit', with examples such as '5 minutes' and '30 seconds'. The value '12 minutes' follows this format precisely: it consists of a number followed by a space and a valid time unit.",
+                    "explanation": "The parameter value '12 minutes' aligns perfectly with the required type, structure, and unit conventions defined in the spec. While not explicitly given as an example, it is fully consistent with the format and requires no normalization or interpretation.",
+                    "output": 5,
+                    "confidence": 0.99,
+                    "correction": {}
+                }
+            },
+            {
+                "user_kwargs": {
+                    "conversation_context": [
+                        {
+                            "role": "user",
+                            "content": "Start a countdown for 5 minutes."
+                        }
+                    ],
+                    "tool_inventory": [
+                        {
+                            "type": "function",
+                            "function": {
+                                "name": "set_timer",
+                                "description": "Set a countdown timer",
+                                "parameters": {
+                                    "type": "object",
+                                    "properties": {
+                                        "duration": {
+                                            "type": "string",
+                                            "description": "Duration in the format 'number + unit' (e.g., '5 minutes', '30 seconds')"
+                                        }
+                                    },
+                                    "required": [
+                                        "duration"
+                                    ]
+                                }
+                            }
+                        }
+                    ],
+                    "tool_call": {
+                        "id": "call_001",
+                        "type": "function",
+                        "function": {
+                            "name": "set_timer",
+                            "arguments": "{ \"duration\": \"300000\" }"
+                        }
+                    },
+                    "parameter_name": "duration",
+                    "parameter_value": "300000"
+                },
+                "output": {
+                    "evidence": "The tool specification defines 'duration' as a string following the 'number + unit' format (e.g., '5 minutes'). The value '300000' is a string, but it lacks a time unit and does not follow the documented pattern.",
+                    "explanation": "The value is likely intended to represent milliseconds, but the format is both undocumented and inconsistent with the specification. It fails both unit and pattern requirements, and cannot be reliably interpreted as valid input without assumptions or external conventions.",
+                    "output": 2,
+                    "confidence": 0.95,
+                    "correction": {
+                        "reason_types": [
+                            "FORMAT_ERROR",
+                            "UNIT_ERROR"
+                        ],
+                        "reasons": "Missing explicit time unit and nonconformant format. Specification requires 'number + unit' format, which this value does not follow.",
+                        "parameter": {
+                            "duration": "5 minutes"
+                        }
+                    }
+                }
+            },
+            {
+                "user_kwargs": {
+                    "conversation_context": [
+                        {
+                            "role": "user",
+                            "content": "Set a timer for five minutes please."
+                        }
+                    ],
+                    "tool_inventory": [
+                        {
+                            "type": "function",
+                            "function": {
+                                "name": "set_timer",
+                                "description": "Set a countdown timer",
+                                "parameters": {
+                                    "type": "object",
+                                    "properties": {
+                                        "duration": {
+                                            "type": "string",
+                                            "description": "Duration in the format 'number + unit' (e.g., '5 minutes', '30 seconds')"
+                                        }
+                                    },
+                                    "required": [
+                                        "duration"
+                                    ]
+                                }
+                            }
+                        }
+                    ],
+                    "tool_call": {
+                        "id": "call_001",
+                        "type": "function",
+                        "function": {
+                            "name": "set_timer",
+                            "arguments": "{ \"duration\": \"5Minutes\" }"
+                        }
+                    },
+                    "parameter_name": "duration",
+                    "parameter_value": "5Minutes"
+                },
+                "output": {
+                    "evidence": "The specification explicitly defines the duration format as 'number + unit' with examples like '5 minutes'. The given value '5Minutes' omits the required space, resulting in a concatenated string.",
+                    "explanation": "The input conveys the intended meaning and includes all required semantic elements (number and unit), but the lack of a space between them breaks the documented pattern. This deviation is moderate-it introduces a risk of failure in strict parsers that expect a space-separated format.",
+                    "output": 3,
+                    "confidence": 0.89,
+                    "correction": {
+                        "reason_types": [
+                            "FORMAT_ERROR"
+                        ],
+                        "reasons": "Value contains both components but violates the explicit formatting requirement of space separation.",
+                        "parameter": {
+                            "duration": "5 minutes"
+                        }
+                    }
+                }
+            }
+        ]
+    }
+]

ibm_watsonx_gov/providers/llmevalkit/function_calling/metrics/trajectory/__init__.py

@@ -0,0 +1,7 @@
+"""Trajectory reflection metrics."""
+from .trajectory import (
+    get_trajectory_reflection_prompt,
+    TrajectoryReflectionPrompt,
+)
+
+__all__ = ["get_trajectory_reflection_prompt", "TrajectoryReflectionPrompt"]

ibm_watsonx_gov/providers/llmevalkit/function_calling/metrics/trajectory/trajectory.py

@@ -0,0 +1,43 @@
+from typing import Any, Dict, List, Union
+
+from llmevalkit.function_calling.metrics.base import FunctionMetricsPrompt
+
+_trajectory_system = (
+    "### Task Description and Role:\n\n"
+    "{{ task_description }}\n\n"
+    "Your output must conform to the following JSON schema, in the same order as the fields appear in the schema:\n"
+    "{{ metric_jsonschema }}"
+)
+
+_trajectory_user: str = (
+    "End-to-end user to agent interaction:\n"
+    "{{ trajectory }}\n\n"
+    "Tool Specification:\n"
+    "{{ tool_inventory }}\n\n"
+    "Return a JSON object as specified in the system prompt. You MUST keep the same order of fields in the JSON object as provided in the JSON schema and examples."
+)
+
+
+class TrajectoryReflectionPrompt(FunctionMetricsPrompt):
+    """Prompt builder for trajectory reflection metrics."""
+
+    system_template = _trajectory_system
+    user_template = _trajectory_user
+
+
+def get_trajectory_reflection_prompt(
+    prompt: TrajectoryReflectionPrompt,
+    trajectory: Union[str, List[Dict[str, str]]],
+    tool_inventory: List[Dict[str, Any]],
+) -> List[Dict[str, str]]:
+    """
+    Build the messages for a trajectory reflection evaluation.
+
+    Returns the list of chat messages (system -> [few-shot] -> user).
+    """
+    return prompt.build_messages(
+        user_kwargs={
+            "trajectory": trajectory,
+            "tool_inventory": tool_inventory,
+        }
+    )