adaptive-harmony 0.1.23__py3-none-any.whl

adaptive_harmony/graders/answer_relevancy_judge/prompts.py

@@ -0,0 +1,58 @@
+SYSTEM = """You are an expert data reviewer.
+You will be given a list of statements.
+You task is to determine whether each statement is relevant to addressing the user's input.
+
+Since you are going to generate a verdict for each statement, the number of `verdicts` SHOULD BE STRICTLY EQUAL to the number of `statements`, and verdicts should be in the same order as the original statements.
+
+You always output a JSON object with the following schema, and nothing else before or after:
+{json_schema}
+
+Examples:
+{shots}
+"""
+
+USER = """Your real task:
+INPUT
+{user_question}
+
+STATEMENTS
+{statements}
+
+```json"""
+
+
+DEFAULT_SHOTS = """INPUT
+What percentage is considered a good rental yield?
+
+STATEMENTS
+0: How are you doing today?
+1: Rental yield is how much you could expect to receive in rent each year from your buy to let investment.
+2: Rental yield is expressed as a percentage - reflecting your rental income against the property's market value.
+3: Anything around the 5-6% mark could be considered a good rental yield.
+4: Anything above 6% could be considered a very good rental yield.
+
+```json
+{
+    "verdicts": [
+        {
+            "reason": "The statement is unrelated to the input.",
+            "score": 0
+        },
+        {
+            "reason": "While the statement discusses rental yields, it does not indicate what constitutes a good rental yield.",
+            "score": 0
+        },
+        {
+            "reason": "While the statement mentions that yield is expressed as a percentage, it does not address the user question.",
+            "score": 0
+        },
+        {
+            "reason": "The statement addresses the user input, specifying what a good rental yield is.",
+            "score": 1
+        },
+        {
+            "reason": "The statement addresses the user input, specifying what a very good rental yield is.",
+            "score": 1
+        },
+    ]
+}```"""

adaptive_harmony/graders/base_grader.py

@@ -0,0 +1,265 @@
+import statistics
+from abc import ABC, abstractmethod
+from typing import Any, Awaitable, Callable, cast
+
+from harmony_client import InferenceModel
+from harmony_client.runtime.data import (
+    AdaptiveGrader,
+    CustomJudge,
+    PrebuiltConfigKey,
+    PrebuiltJudge,
+    RemoteRewardEndpoint,
+)
+
+from adaptive_harmony import Grade, StringThread
+from adaptive_harmony.graders.utils import FailedJudgeLog, SuccessJudgeLog
+from adaptive_harmony.logging_table import Table
+from adaptive_harmony.parameters import Model
+from adaptive_harmony.runtime import RecipeContext
+
+
+class BaseGrader[LogType](ABC):
+    """
+    Base Grader to inherit from when building a scoring function.
+    """
+
+    def __init__(self, grader_key: str):
+        self._logs: list[LogType] = []
+        self.grader_key = grader_key
+
+    @abstractmethod
+    async def grade(self, sample: StringThread) -> Grade:
+        """
+        Grade a single sample.
+        Returns a single float score, with optional metadata.
+        Metadata can be useful for evals when LLM reasoning regarding the score is available.
+        """
+        pass
+
+    async def score_float_value(self, sample: StringThread) -> float:
+        """Returns only the float score from .score"""
+        return (await self.grade(sample)).value
+
+    def add_log(self, log_data: LogType) -> None:
+        """Add a log entry to the scorer's log collection."""
+        self._logs.append(log_data)
+
+    def get_logs(self, clear: bool = False, log_all_samples: bool = False) -> dict[str, float | Table]:
+        """
+        Get aggregated logs from all score calls.
+        Base implementation computes statistics for "score" keys in individual logs.
+        If there are none, returns empty dict.
+        """
+        if not self._logs:
+            return {}
+
+        scores = [s for s in [cast(dict[str, Any], log).get("score") for log in self._logs] if s is not None]
+        logs = {}
+        if scores:
+            logs.update(
+                dict(
+                    **{
+                        f"score/{key}": value
+                        for key, value in dict(
+                            mean=statistics.mean(scores),
+                            std=statistics.stdev(scores) if len(scores) > 1 else 0.0,
+                            min=min(scores),
+                            max=max(scores),
+                            count=len(scores),
+                        ).items()
+                    },
+                )
+            )
+        if clear:
+            self.clear_logs()
+        return logs
+
+    def clear_logs(self) -> None:
+        """
+        Clear all accumulated logs.
+        """
+        self._logs.clear()
+
+    def get_sample_tables(
+        self, successful_samples: list[SuccessJudgeLog], failed_samples: list[FailedJudgeLog] | None = None
+    ):
+        table_logs = {}
+        scored_samples = (
+            Table()
+            .add_column("Prompt", [log["prompt"] for log in successful_samples])
+            .add_column("Reasoning", [log.get("reasoning") for log in successful_samples])
+            .add_column("Score", [float(log["score"]) for log in successful_samples])
+        )
+        if failed_samples:
+            unscored_samples = (
+                Table()
+                .add_column("Prompt", [log.get("prompt") for log in failed_samples])
+                .add_column("Error", [str(log["error"]) for log in failed_samples])
+            )
+            table_logs["score/unscored_samples"] = unscored_samples
+        table_logs["score/scored_samples"] = scored_samples
+        table_logs["score/unscored_samples_count"] = len(failed_samples) if failed_samples else 0
+        table_logs["score/scored_samples_count"] = len(successful_samples)
+        return table_logs
+
+    @classmethod
+    def from_function(
+        cls, grader_key: str, async_fn: Callable[[StringThread], Awaitable[float]]
+    ) -> "BaseGrader[dict[str, Any]]":
+        class FunctionScorer(BaseGrader[dict[str, float]]):
+            def __init__(self):
+                super().__init__(grader_key)
+
+            async def grade(self, sample: StringThread) -> Grade:
+                result = await async_fn(sample)
+                grade = Grade(value=result, grader_key=self.grader_key)
+                self.add_log({"score": result})
+                return grade
+
+        return FunctionScorer()
+
+    @classmethod
+    async def from_config(
+        cls,
+        grader_config: AdaptiveGrader,
+        ctx: RecipeContext,
+        tp: int | None = None,
+        kv_cache_len: int | None = None,
+        max_tokens: int | None = None,
+    ) -> "BaseGrader[dict[str, Any]]":
+        match grader_config.config.type:
+            case "Judge":
+                config = cast(CustomJudge, grader_config.config)
+                return await cls.from_templated_judge(
+                    grader_config.key, str(grader_config.grader_id), config, ctx, tp, kv_cache_len, max_tokens
+                )
+            case "Prebuilt":
+                config = cast(PrebuiltJudge, grader_config.config)
+                return await cls.from_prebuilt_judge(
+                    grader_config.key, str(grader_config.grader_id), config, ctx, tp, kv_cache_len
+                )
+            case "Remote":
+                config = cast(RemoteRewardEndpoint, grader_config.config)
+                return cls.from_remote_reward_endpoint(grader_config.key, str(grader_config.grader_id), config)
+            case _:
+                raise ValueError(f"Invalid grader type: {grader_config.config.type}")
+
+    @classmethod
+    async def from_templated_judge(
+        cls,
+        grader_key: str,
+        grader_id: str,
+        config: CustomJudge,
+        ctx: RecipeContext,
+        tp: int | None = None,
+        kv_cache_len: int | None = None,
+        max_tokens: int | None = None,
+    ) -> "BaseGrader[dict[str, Any]]":
+        # Import here to avoid circular dependency
+        from adaptive_harmony.graders.templated_prompt_judge import (
+            BinaryJudgeOutput,
+            TemplatedPromptJudgeGrader,
+        )
+
+        # Convert examples to template variables
+        examples = []
+        for example in config.examples:
+            examples.append(
+                {
+                    "context_str": (
+                        "\n".join(f"{msg.role}:\n{msg.content}" for msg in example.input[:-1])
+                        if len(example.input) > 1
+                        else ""
+                    ),
+                    "user_question": example.input[-1].content if example.input else "",
+                    "completion": example.output,
+                    "output_json": f'{{"reasoning": "{example.reasoning or ""}", "score": "{"PASS" if example.pass_ else "FAIL"}"}}',
+                }
+            )
+
+        template_vars = {
+            "criteria": config.criteria,
+            "examples": examples,
+        }
+
+        model = await get_model(ctx, grader_key, config.model_key, kv_cache_len, max_tokens, tp)
+
+        return TemplatedPromptJudgeGrader(
+            grader_key=grader_key,
+            model=model,
+            grader_id=grader_id,
+            system_template=config.system_template,
+            user_template=config.user_template,
+            output_model=BinaryJudgeOutput,
+            template_variables=template_vars,
+        )  # type: ignore[return-value]
+
+    @classmethod
+    async def from_prebuilt_judge(
+        cls,
+        grader_key: str,
+        grader_id: str,
+        config: PrebuiltJudge,
+        ctx: RecipeContext,
+        tp: int | None = None,
+        kv_cache_len: int | None = None,
+        max_tokens: int | None = None,
+    ) -> "BaseGrader[dict[str, Any]]":
+        model = await get_model(ctx, grader_key, config.model_key, kv_cache_len, max_tokens, tp)
+
+        match config.prebuilt_config_key:
+            case PrebuiltConfigKey.Faithfulness:
+                # Import here to avoid circular dependency
+                from adaptive_harmony.graders.faithfulness_judge.faithfulness_judge import FaithfulnessGrader
+
+                return FaithfulnessGrader(
+                    grader_key=grader_key,
+                    grader_id=grader_id,
+                    model=model,
+                )
+            case PrebuiltConfigKey.AnswerRelevancy:
+                # Import here to avoid circular dependency
+                from adaptive_harmony.graders.answer_relevancy_judge.answer_relevancy_judge import AnswerRelevancyGrader
+
+                return AnswerRelevancyGrader(
+                    grader_key=grader_key,
+                    grader_id=grader_id,
+                    model=model,
+                )
+            case PrebuiltConfigKey.ContextRelevancy:
+                # Import here to avoid circular dependency
+                from adaptive_harmony.graders.context_relevancy_judge.context_relevancy_judge import (
+                    ContextRelevancyGrader,
+                )
+
+                return ContextRelevancyGrader(
+                    grader_key=grader_key,
+                    grader_id=grader_id,
+                    model=model,
+                )
+            case _:
+                raise ValueError(f"Invalid prebuilt judge type: {config.prebuilt_config_key}")
+
+    @classmethod
+    def from_remote_reward_endpoint(
+        cls, grader_key: str, grader_id: str, config: RemoteRewardEndpoint
+    ) -> "BaseGrader[dict[str, Any]]":
+        # Import here to avoid circular dependency
+        from adaptive_harmony.graders.reward_server_grader import RewardServerGrader
+
+        return RewardServerGrader(grader_key=grader_key, grader_id=grader_id, reward_server_ip=config.url)
+
+
+async def get_model(
+    ctx: RecipeContext,
+    grader_key: str,
+    model_key: str,
+    kv_cache_len: int | None,
+    max_tokens: int | None,
+    tp: int | None,
+) -> InferenceModel:
+    model_builder = await Model(model_key=model_key).to_builder(
+        ctx, kv_cache_len=kv_cache_len, tokens_to_generate=max_tokens, tp=tp
+    )
+    model = await model_builder.spawn_inference(grader_key)
+    return model

adaptive_harmony/graders/binary_judge/__init__.py

@@ -0,0 +1,8 @@
+from .binary_judge import BinaryJudgeGrader, BinaryJudgeOutput
+from .prompts import BinaryJudgeShot
+
+__all__ = [
+    "BinaryJudgeGrader",
+    "BinaryJudgeOutput",
+    "BinaryJudgeShot",
+]

adaptive_harmony/graders/binary_judge/binary_judge.py

@@ -0,0 +1,202 @@
+from random import shuffle
+from typing import Literal, TypedDict
+
+from pydantic import BaseModel, Field
+
+from adaptive_harmony import InferenceModel, StringThread
+from adaptive_harmony.core.reasoning import remove_reasoning
+from adaptive_harmony.core.structured_output import JsonParseError, render_pydantic_model, render_schema
+from adaptive_harmony.core.utils import SingleTurnShot, stringify_thread
+from adaptive_harmony.graders import BaseGrader, Grade
+from adaptive_harmony.graders.binary_judge.prompts import DEFAULT_SHOTS, SYSTEM, USER, BinaryJudgeShot
+from adaptive_harmony.graders.exceptions import IgnoreScoreException
+from adaptive_harmony.graders.utils import (
+    FailedJudgeLog,
+    SuccessJudgeLog,
+    separate_context_from_last_user_turn,
+    validate_thread_last_assistant,
+)
+from adaptive_harmony.logging_table import Table
+
+
+class BinaryJudgeOutput(BaseModel):
+    reasoning: str = Field(description="Reasoning to support the rationale behind the score")
+    score: Literal["PASS", "FAIL", "NA"] = Field(description="The score for the sample")
+
+
+class ScoresMap(TypedDict):
+    PASS: float
+    FAIL: float
+
+
+OPENAI_MODEL_FAMILIES_TEMPERATURE_1_ONLY = ["gpt-5", "o1", "o3", "o4"]
+
+
+class BinaryJudgeGrader(BaseGrader):
+    """
+    Binary judge for scoring samples as PASS, FAIL or NA using few-shot prompting.
+    If custom shots are provided, they are used instead of the default shots.
+    """
+
+    def __init__(
+        self,
+        grader_key: str,
+        model: InferenceModel,
+        criteria: str,
+        shots: list[BinaryJudgeShot] | None = None,
+        temperature: float = 0.0,
+        grader_id: str | None = None,
+    ):
+        super().__init__(grader_key)
+        self._logs: list[SuccessJudgeLog | FailedJudgeLog] = []  # type: ignore[assignment]
+        self.criteria = criteria
+        self.model = model
+        self.temperature = temperature
+        # Set temperature to 1.0 if model_key is an OpenAI model in the temperature-1-only list
+        model_path: str = model.get_builder_args().get("path")  # type: ignore[assignment]
+        if model_path.startswith("openai://"):
+            model_name = model_path.removeprefix("openai://").split("?")[0]
+            if any(model_name.startswith(model) for model in OPENAI_MODEL_FAMILIES_TEMPERATURE_1_ONLY):
+                temperature = 1.0
+        self.model = model.temperature(temperature)
+        # Score mapping
+        self.scores_map: ScoresMap = {"PASS": 1.0, "FAIL": 0.0}
+        self.grader_id_or_key = grader_id or grader_key
+
+        self._original_shots = shots or DEFAULT_SHOTS
+        self._shots = self._format_user_shots(shots or DEFAULT_SHOTS)
+
+    @property
+    def shots(self) -> list[BinaryJudgeShot]:
+        return self._original_shots
+
+    @shots.setter
+    def shots(self, shots: list[BinaryJudgeShot]):
+        self._original_shots = shots
+        self._shots = self._format_user_shots(shots)
+
+    @staticmethod
+    def _extract_user_template_kwargs(thread: StringThread) -> dict[str, str]:
+        validate_thread_last_assistant(thread)
+        # Separate conversation context from last user turn
+        context_turns, user_question = separate_context_from_last_user_turn(thread)
+        context_str = stringify_thread(StringThread(context_turns))
+        completion = remove_reasoning(thread.last_content())
+
+        assert user_question, "There must be at least one user turn"
+        return dict(
+            context=context_str,
+            user_question=user_question,
+            completion=completion,
+        )
+
+    def _get_placeholder_reasoning(self, score: Literal["PASS", "FAIL", "NA"]) -> str:
+        if score == "PASS":
+            return "The completions complies with the criteria"
+        elif score == "FAIL":
+            return "The completion does not comply with the criteria"
+        else:
+            return "The criteria is not applicable to the completion"
+
+    def _format_user_shots(self, shots: list[BinaryJudgeShot]) -> list[SingleTurnShot]:
+        """
+        Turn a possibly multi turn example into a single turn one,
+        with appropriate kwargs to format the task's prompt templates
+        """
+        new_shots: list[SingleTurnShot] = []
+        for shot in shots:
+            placeholder_reasoning = self._get_placeholder_reasoning(shot.score)
+
+            user_template_kwargs = self._extract_user_template_kwargs(shot.thread)
+            user_template_kwargs["criteria"] = shot.criteria or self.criteria
+            single_turn_shot = SingleTurnShot(
+                user=user_template_kwargs,
+                assistant={
+                    "json_answer": render_pydantic_model(
+                        BinaryJudgeOutput(
+                            reasoning=shot.reasoning or placeholder_reasoning,
+                            score=shot.score,
+                        )
+                    )
+                },
+            )
+            new_shots.append(single_turn_shot)
+
+        return new_shots
+
+    def _get_judge_prompt(self, thread: StringThread) -> StringThread:
+        """Build the judging prompt for a given sample."""
+        # build the real user template kwargs
+        user_template_kwargs = self._extract_user_template_kwargs(thread)
+        user_template_kwargs["criteria"] = self.criteria
+        # system kwarg
+        output_json_schema = render_schema(BinaryJudgeOutput)
+
+        # system
+        prompt = StringThread().system(SYSTEM.format(json_schema=output_json_schema))
+        # shots
+        for shot in self._shots:
+            prompt = prompt.user(USER.format(**shot["user"]))
+            prompt = prompt.assistant(shot["assistant"]["json_answer"])
+        # real input
+        prompt = prompt.user(USER.format(**user_template_kwargs))
+
+        return prompt
+
+    async def grade(self, sample: StringThread) -> Grade:
+        judging_prompt = self._get_judge_prompt(sample)
+        str_prompt = stringify_thread(judging_prompt, sep=f"\n\n{'-' * 10}\n\n")
+
+        try:
+            _, parsed_output = await self.model.generate_and_validate(judging_prompt, BinaryJudgeOutput)
+        except JsonParseError as e:
+            self.add_log({"prompt": str_prompt, "error": f"{str(e)}\n\nCOMPLETION:\n{e.completion}"})
+            raise
+        except Exception as e:
+            self.add_log({"prompt": str_prompt, "error": str(e)})
+            raise
+
+        float_score = self.scores_map.get(parsed_output.score)
+
+        # NA case, ignore score
+        if float_score is None:
+            self.add_log({"prompt": str_prompt, "error": f"Non applicable score: {parsed_output.reasoning}"})
+            raise IgnoreScoreException(f"Non applicable score: {parsed_output.reasoning}")
+
+        else:
+            grade = Grade(value=float_score, grader_key=self.grader_id_or_key, reasoning=parsed_output.reasoning)
+            self.add_log({"score": float_score, "prompt": str_prompt, "reasoning": parsed_output.reasoning})
+
+            return grade
+
+    def add_log(self, log: SuccessJudgeLog | FailedJudgeLog) -> None:  # type: ignore[override]
+        self._logs.append(log)
+
+    def get_logs(self, clear: bool = False, log_all_samples: bool = False) -> dict[str, float | Table]:
+        # Only clear logs at the end if clear is True
+        logs = super().get_logs(clear=False)
+
+        # get sample of PASS and FAIL samples to log in table
+        successfully_scored_samples = [log for log in self._logs if "score" in log]
+        if not log_all_samples:
+            shuffle(successfully_scored_samples)
+            samples_score_0 = [log for log in successfully_scored_samples if log["score"] == self.scores_map["FAIL"]][
+                :5
+            ]
+            samples_score_1 = [log for log in successfully_scored_samples if log["score"] == self.scores_map["PASS"]][
+                :5
+            ]
+            subset_successfully_scored_samples = samples_score_0 + samples_score_1
+        else:
+            subset_successfully_scored_samples = successfully_scored_samples
+
+        # get failed samples to log in table
+        failed_scored_samples = [log for log in self._logs if "error" in log]
+
+        sample_logs = self.get_sample_tables(subset_successfully_scored_samples, failed_scored_samples)
+        logs.update(sample_logs)
+
+        if clear:
+            self.clear_logs()
+
+        return logs

adaptive_harmony/graders/binary_judge/prompts.py

@@ -0,0 +1,125 @@
+from typing import Literal, NamedTuple
+
+from harmony_client.runtime.data import CustomJudgeExample
+
+from adaptive_harmony import StringThread
+
+SYSTEM = """You are an expert evaluator that evaluates completions generated by an AI model on a fixed criterion.
+You will be given all elements of an interaction between human an AI model:
+    The full context of the conversation so far leading up to the last user turn/question is under the CONTEXT header. It may contain extra contextual information.
+    The last user turn/question is under the USER QUESTION header. It may contain extra contextual information.
+    The model's completion is under the COMPLETION TO EVALUATE header.
+    The evaluation criterion is under the EVALUATION CRITERION section.
+
+In order to to analyze and score a completion, you always run the following steps without exception:
+First, you read the CONTEXT, USER QUESTION and COMPLETION TO EVALUATE.
+Then, you analyze the COMPLETION TO EVALUATE, and assign it a PASS, FAIL or NA score according to the criterion: FAIL if the completion does not meet the criterion, PASS if it does, and NA if the criterion is not applicable to the example. These are the rules to follow:
+- You must always evaluate the COMPLETION TO EVALUATE based solely on the USER QUESTION, and never on an intermediary question that might have been asked in the CONTEXT. The CONTEXT is there for context only.
+- Do not make any judgement on text that is in the CONTEXT or USER QUESTION; you are evaluating the COMPLETION TO EVALUATE text only.
+- You must not use the original instructions given to the model in the CONTEXT for your judgement. Focus only on the EVALUATION CRITERION, without any other influencing factors.
+- You are forbidden to return a score other than PASS, FAIL or NA for each criterion.
+- If the criterion is conditional, and is not applicable to the specific USER QUESTION + COMPLETION TO EVALUATE pair, you must score it as NA.
+- Return a single score, no matter how many things are evaluated or contemplated in the criterion. A PASS means the completion complied with everything.
+
+Finally, output an explanation for your judgement and the score for the criterion, as exemplified below. The output should be a well-formatted JSON string that conforms to the JSON schema below. Do not output anything else other than the JSON string.
+
+Here is the output JSON schema you must strictly follow, with field descriptions and value types. All fields are required.
+{json_schema}
+
+Evaluate only the final COMPLETION TO EVALUATE with regard to the USER QUESTION shown. Do not return any preamble or explanations, Return exactly one JSON string"""
+
+
+USER = """CONTEXT
+{context}
+
+USER QUESTION
+{user_question}
+
+COMPLETION TO EVALUATE
+{completion}
+
+EVALUATION CRITERION
+{criteria}
+
+OUTPUT
+"""
+
+
+class BinaryJudgeShot(NamedTuple):
+    """
+    Example shot for BinaryJudgeScorer.
+    If no criteria is specified, we assume the rest
+    of the shot relates to the criteria passed to the BinaryJudgeScorer.
+    """
+
+    thread: StringThread
+    reasoning: str | None
+    score: Literal["PASS", "FAIL", "NA"]
+    criteria: str | None = None
+
+
+def to_binary_judge_shot(example: CustomJudgeExample) -> BinaryJudgeShot:
+    return BinaryJudgeShot(
+        thread=StringThread([(turn.role, turn.content) for turn in example.input]).assistant(example.output),
+        reasoning=example.reasoning,
+        score="PASS" if example.pass_ else "FAIL",
+    )
+
+
+DEFAULT_SHOTS = [
+    BinaryJudgeShot(
+        thread=StringThread(
+            [
+                (
+                    "system",
+                    "You are a savy boating conosseur. You know everything there is to know about boats. Your responses to questions must be quick and short, almost dismissive.",
+                ),
+                ("user", " Hey, how are you doing?"),
+                ("assistant", "Doing great!"),
+                ("user", "Perfect. Tell me, how many masts does a ketch have?"),
+                ("assistant", "A ketch has two masts"),
+            ]
+        ),
+        reasoning="The completion is in English instead of french.",
+        score="FAIL",
+        criteria="The completion must be in french",
+    ),
+    BinaryJudgeShot(
+        thread=StringThread(
+            [
+                (
+                    "system",
+                    "You are a customer service assistant. You can use informal tone if the user is informal with you, but always make sure the customer's problem is fixed.",
+                ),
+                (
+                    "user",
+                    "Write a reply to a customer called John Smith that has requested for a refund of our product. Our company policy is to provide refunds up to 30 days after the purchase.",
+                ),
+                (
+                    "assistant",
+                    "Dear John Smith,\nThank you for reaching out regarding your request for a refund. We appreciate your business and understand that sometimes products may not meet expectations.\nAs per our company policy, we are happy to provide refunds within 30 days of your purchase. If your request falls within this timeframe and the product is in its original condition, we will process your refund promptly. Please ensure that you return the item to us along with proof of purchase.\nIf you have any questions or need assistance with the return process, feel free to contact us.",
+                ),
+            ]
+        ),
+        reasoning="The completion is in general polite to the customer.",
+        score="PASS",
+        criteria="Completion must be polite, with use of proper language and no slang.",
+    ),
+    BinaryJudgeShot(
+        thread=StringThread(
+            [
+                (
+                    "system",
+                    "You are a savy boating conosseur. You know everything there is to know about boats. Your responses to questions must be quick and short, almost dismissive.",
+                ),
+                ("user", " Hey, how are you doing?"),
+                ("assistant", "Doing great!"),
+                ("user", "Perfect. Tell me, how many masts does a ketch have?"),
+                ("assistant", "A ketch has two masts"),
+            ]
+        ),
+        reasoning="The user did not ask about hunting, criterion is not applicable",
+        score="NA",
+        criteria="If the user is asking about hunting topics, state that you cannot answer.",
+    ),
+]