sglang 0.2.11__py3-none-any.whl → 0.2.12__py3-none-any.whl

sglang/test/simple_eval_humaneval.py

@@ -6,21 +6,15 @@ Mark Chen and Jerry Tworek and Heewoo Jun and Qiming Yuan and Henrique Ponde de
 https://arxiv.org/abs/2107.03374 https://github.com/openai/human-eval/ 
 """
 
-import json
-import logging
-import multiprocessing
 import random
 import re
-from collections import Counter, defaultdict
 from concurrent.futures import ThreadPoolExecutor, as_completed
-from io import BytesIO
-from typing import Any, Dict, List, Tuple
+from typing import Dict, List
 
-import blobfile as bf
 import tqdm
 
 try:
-    from human_eval.data import HUMAN_EVAL, read_problems
+    from human_eval.data import read_problems
     from human_eval.evaluation import estimate_pass_at_k
     from human_eval.execution import check_correctness  # , unsafe_execute
 except (ImportError, ModuleNotFoundError):

sglang/test/simple_eval_mgsm.py

@@ -0,0 +1,203 @@
+# Adapted from https://github.com/openai/simple-evals/
+
+"""
+MGSM: Multilingual Grade School Math Benchmark (MGSM) is a benchmark of grade-school math problems. 
+Language Models are Multilingual Chain-of-Thought Reasoners
+Freda Shi, Mirac Suzgun, Markus Freitag, Xuezhi Wang, Suraj Srivats, Soroush Vosoughi, Hyung Won Chung, Yi Tay, Sebastian Ruder, Denny Zhou, Dipanjan Das, Jason Wei
+https://arxiv.org/abs/2210.03057 reference: https://github.com/google-research/url-nlp 
+"""
+
+import re
+import urllib
+from typing import Optional
+
+from sglang.test import simple_eval_common as common
+from sglang.test.simple_eval_common import (
+    HTML_JINJA,
+    Eval,
+    EvalResult,
+    SamplerBase,
+    SingleEvalResult,
+)
+
+ALL_LANGUAGES = ["bn", "de", "en", "es", "fr", "ja", "ru", "sw", "te", "th", "zh"]
+LATIN_LANGUAGES = ["de", "en", "es", "fr", "sw"]
+NON_LATIN_LANGUAGES = ["bn", "ja", "ru", "te", "th", "zh"]
+
+LANG_TO_FPATH = {
+    "bn": "https://openaipublic.blob.core.windows.net/simple-evals/mgsm_bn.tsv",
+    "de": "https://openaipublic.blob.core.windows.net/simple-evals/mgsm_de.tsv",
+    "en": "https://openaipublic.blob.core.windows.net/simple-evals/mgsm_en.tsv",
+    "es": "https://openaipublic.blob.core.windows.net/simple-evals/mgsm_es.tsv",
+    "fr": "https://openaipublic.blob.core.windows.net/simple-evals/mgsm_fr.tsv",
+    "ja": "https://openaipublic.blob.core.windows.net/simple-evals/mgsm_ja.tsv",
+    "ru": "https://openaipublic.blob.core.windows.net/simple-evals/mgsm_ru.tsv",
+    "sw": "https://openaipublic.blob.core.windows.net/simple-evals/mgsm_sw.tsv",
+    "te": "https://openaipublic.blob.core.windows.net/simple-evals/mgsm_te.tsv",
+    "th": "https://openaipublic.blob.core.windows.net/simple-evals/mgsm_th.tsv",
+    "zh": "https://openaipublic.blob.core.windows.net/simple-evals/mgsm_zh.tsv",
+}
+LANG_TO_INSTRUCTIONS = {
+    "en": """Solve this math problem. Give the reasoning steps before giving the final answer on the last line by itself in the format of "Answer:". Do not add anything other than the integer answer after "Answer:".
+
+{input}""",
+    "bn": """এই গণিতের সমস্যাটি সমাধান করুন। চূড়ান্ত উত্তর দেওয়ার আগে যুক্তিসম্পন্ন পদক্ষেপ প্রদান করুন। চূড়ান্ত উত্তরটি একক সংখ্যা হিসাবে "উত্তর:" এর পরে শেষ লাইনে দিন। "উত্তর:" এর পরে অন্য কিছু যুক্ত করবেন না।.
+
+{input}""",
+    "de": """Löse dieses Mathematikproblem. Gib die Schritte zur Begründung an, bevor du die endgültige Antwort in der letzten Zeile alleine im Format "Antwort:" gibst. Füge nichts anderes als die ganzzahlige Antwort nach "Antwort:" hinzu.
+
+{input}""",
+    "es": """Resuelve este problema matemático. Proporciona los pasos de razonamiento antes de dar la respuesta final en la última línea por sí misma en el formato de "Respuesta:". No añadas nada más que la respuesta entera después de "Respuesta:".
+
+{input}""",
+    "fr": """Résolvez ce problème de mathématiques. Donnez les étapes de raisonnement avant de fournir la réponse finale sur la dernière ligne elle-même dans le format de "Réponse:". N'ajoutez rien d'autre que la réponse entière après "Réponse:".
+
+{input}""",
+    "ja": """の数学の問題を解いてください。最終的な答えを出す前に、解答の推論過程を記述してください。そして最後の行には "答え:" の形式で答えを記述し、その後には整数の答え以外何も追加しないでください。
+
+{input}""",
+    "ru": """Решите эту математическую задачу. Объясните шаги рассуждения перед тем, как дать окончательный ответ в последней строке сам по себе в формате "Ответ:". Не добавляйте ничего, кроме целочисленного ответа после "Ответ:".
+
+{input}""",
+    "sw": """Suluhisha tatizo hili la hesabu. Toa hatua za mantiki kabla ya kutoa jibu la mwisho kwenye mstari wa mwisho peke yake katika muundo wa "Jibu:". Usiongeze chochote kingine isipokuwa jibu la integer baada ya "Jibu:".
+
+{input}""",
+    "te": """ఈ గణిత సమస్యను పరిష్కరించండి. చివరి సమాధానాన్ని ఇవ్వదానికి ముందు తర్కాత్మక అదుగులను ఇవ్వండి. చివరి పంక్తిలో మాత్రమే 'సమాధానం:' అనే ఆకారంలో చివరి సమాధానాద్ని ఇవ్వండి సమాధానం: తర్వాత పూర్ణాంక సమాధానానికి తప్పించి ఎదేనా చేర్చవద్దు.
+
+{input}""",
+    "th": """แก้ปัญหาคณิตศาสตร์นี้ ให้ให้ขั้นตอนการใช้เหตุผลก่อนที่จะให้คำตอบสุดท้ายในบรรทัดสุดท้ายโดยอยู่ในรูปแบบ "คำตอบ:" ไม่ควรเพิ่มอะไรนอกจากคำตอบที่เป็นจำนวนเต็มหลังจาก "คำตอบ:"
+
+{input}""",
+    "zh": """解决这个数学问题。在最后一行给出答案前，请提供推理步骤。最后一行应该以 "答案: " 的形式独立给出答案。在 "答案：" 后不要添加除整数答案之外的任何内容。
+
+{input}""",
+}
+
+LANG_TO_ANSWER_PREFIX = {
+    "en": "Answer",
+    "bn": "উত্তর",
+    "de": "Antwort",
+    "es": "Respuesta",
+    "fr": "Réponse",
+    "ja": "答え",
+    "ru": "Ответ",
+    "sw": "Jibu",
+    "te": "సమాధానం",
+    "th": "คำตอบ",
+    "zh": "答案",
+}
+
+
+def parse_answer(answer: str, answer_prefix: str) -> str:
+    if answer_prefix not in answer:
+        return ""
+
+    answer_text = answer.split(answer_prefix)[-1].strip()
+
+    # find all the numbers (including decimals) in the string
+    numbers = re.findall(r"\d+\.?\d*", answer_text.replace(",", ""))
+
+    # return the first number (removing trailing decimal point if present),
+    # or an empty string if there were no numbers
+    return numbers[-1].rstrip(".") if numbers else ""
+
+
+def score_mgsm(target: str, prediction: str) -> bool:
+    if "." in prediction:
+        prediction = prediction.rstrip("0").rstrip(".")
+
+    target = target.replace(",", "")
+    prediction = prediction.replace(",", "")
+
+    return target == prediction
+
+
+def get_lang_examples(lang: str) -> list[dict[str, str]]:
+    fpath = LANG_TO_FPATH[lang]
+    examples = []
+    with urllib.request.urlopen(fpath) as f:
+        for line in f.read().decode("utf-8").splitlines():
+            inputs, targets = line.strip().split("\t")
+            if "." in targets:
+                raise ValueError(f"targets {targets} contains a decimal point.")
+            # targets = int(targets.replace(",", ""))
+            examples.append({"inputs": inputs, "targets": targets, "lang": lang})
+    return examples
+
+
+def get_all_examples() -> list[dict[str, str]]:
+    examples = []
+    for lang in ALL_LANGUAGES:
+        if lang != "en":
+            continue
+        examples += get_lang_examples(lang)
+    return examples
+
+
+class MGSMEval(Eval):
+    def __init__(
+        self,
+        num_examples_per_lang: int = 250,  # restrict to a subset of the data for debugging
+        num_threads: int = 64,
+        languages: Optional[list[str]] = ALL_LANGUAGES,
+    ):
+        if languages is None:
+            languages = ALL_LANGUAGES
+        else:
+            for language in languages:
+                if language not in ALL_LANGUAGES:
+                    raise ValueError(
+                        f"language {language} is not a valid language. "
+                        f"It should be one in {ALL_LANGUAGES}"
+                    )
+        self._languages = languages
+        self._num_examples_per_lang = num_examples_per_lang
+        self._num_threads = num_threads
+
+        examples = []
+        for lang in self._languages:
+            lang_examples = get_lang_examples(lang)
+            examples.extend(lang_examples[: self._num_examples_per_lang])
+        self.examples = examples
+
+    def __call__(self, sampler: SamplerBase) -> EvalResult:
+        def fn(example: dict[str, str]):
+            language = example["lang"]
+            latin_language = (
+                "group_latin" if language in LATIN_LANGUAGES else "group_non_latin"
+            )
+            correct_answer = example["targets"]
+            instructoin = LANG_TO_INSTRUCTIONS[language]
+            prompt_messages = [
+                sampler._pack_message(
+                    content=instructoin.format(input=example["inputs"]), role="user"
+                )
+            ]
+            try:
+                response_text = sampler(prompt_messages)
+            except Exception as e:
+                response_text = ""
+
+            answer_prefix = LANG_TO_ANSWER_PREFIX[language]
+            extracted_answer = parse_answer(response_text, answer_prefix)
+
+            score = score_mgsm(correct_answer, extracted_answer)
+            html = common.jinja_env.from_string(HTML_JINJA).render(
+                prompt_messages=prompt_messages,
+                next_message=dict(content=response_text, role="assistant"),
+                score=score,
+                correct_answer=correct_answer,
+                extracted_answer=extracted_answer,
+            )
+            convo = prompt_messages + [dict(content=response_text, role="assistant")]
+            return SingleEvalResult(
+                html=html,
+                score=score,
+                convo=convo,
+                metrics={language: score, latin_language: score},
+            )
+
+        results = common.map_with_progress(
+            fn, self.examples, num_threads=self._num_threads
+        )
+        return common.aggregate_results(results, default_stats=("mean", "std"))

sglang/test/srt/sampling/penaltylib/utils.py

@@ -0,0 +1,337 @@
+import dataclasses
+import enum
+import typing
+import unittest
+
+import torch
+
+from sglang.srt.sampling.penaltylib.orchestrator import (
+    BatchedPenalizerOrchestrator,
+    _BatchedPenalizer,
+    _BatchLike,
+)
+
+
+@dataclasses.dataclass
+class MockSamplingParams:
+    frequency_penalty: float = 0.0
+    min_new_tokens: int = 0
+    stop_token_ids: typing.List[int] = None
+    presence_penalty: float = 0.0
+    repetition_penalty: float = 1.0
+
+
+@dataclasses.dataclass
+class MockTokenizer:
+    eos_token_id: int
+
+
+@dataclasses.dataclass
+class MockReq:
+    origin_input_ids: typing.List[int]
+    sampling_params: MockSamplingParams
+    tokenizer: MockTokenizer
+
+
+class StepType(enum.Enum):
+    INPUT = "input"
+    OUTPUT = "output"
+
+
+@dataclasses.dataclass
+class Step:
+    type: StepType
+    token_ids: typing.List[int]
+    expected_tensors: typing.Dict[str, torch.Tensor]
+    # assume initial logits are all 1
+    expected_logits: torch.Tensor
+
+
+@dataclasses.dataclass
+class Subject:
+    sampling_params: MockSamplingParams
+    # first step must be input, which will be converted to Req
+    steps: typing.List[Step]
+    eos_token_id: int = -1
+
+    def __post_init__(self):
+        if self.steps[0].type != StepType.INPUT:
+            raise ValueError("First step must be input")
+
+        # each steps should have the same expected_tensors.keys()
+        for i in range(1, len(self.steps)):
+            if self.tensor_keys(i) != self.tensor_keys():
+                raise ValueError(
+                    f"Expected tensors keys must be the same for all steps. Got {self.steps[i].expected_tensors.keys()} for key={i} and {self.steps[0].expected_tensors.keys()}"
+                )
+
+    def tensor_keys(self, i: int = 0) -> typing.Set[str]:
+        return set(self.steps[i].expected_tensors.keys())
+
+    def to_req(self) -> MockReq:
+        return MockReq(
+            origin_input_ids=self.steps[0].token_ids,
+            sampling_params=self.sampling_params,
+            tokenizer=MockTokenizer(eos_token_id=self.eos_token_id),
+        )
+
+
+@dataclasses.dataclass
+class Case:
+    enabled: bool
+    test_subjects: typing.List[Subject]
+
+    def __post_init__(self):
+        # each test_subjects.steps should have the same expected_tensors.keys()
+        for i in range(1, len(self.test_subjects)):
+            if self.tensor_keys(i) != self.tensor_keys():
+                raise ValueError(
+                    f"Expected tensors keys must be the same for all test_subjects. Got {self.test_subjects[i].tensor_keys()} for key={i} and {self.test_subjects[0].tensor_keys()}"
+                )
+
+    def tensor_keys(self, i: int = 0) -> typing.List[str]:
+        return set(self.test_subjects[i].tensor_keys())
+
+
+class BaseBatchedPenalizerTest(unittest.TestCase):
+    Penalizer: typing.Type[_BatchedPenalizer]
+    device = "cuda"
+    vocab_size = 5
+
+    enabled: Subject = None
+    disabled: Subject = None
+
+    def setUp(self):
+        if self.__class__ == BaseBatchedPenalizerTest:
+            self.skipTest("Base class for penalizer tests")
+
+        self.create_test_subjects()
+        self.create_test_cases()
+
+    def tensor(self, data, **kwargs) -> torch.Tensor:
+        """
+        Shortcut to create a tensor with device=self.device.
+        """
+        return torch.tensor(data, **kwargs, device=self.device)
+
+    def create_test_subjects(self) -> typing.List[Subject]:
+        raise NotImplementedError()
+
+    def create_test_cases(self):
+        self.test_cases = [
+            Case(enabled=True, test_subjects=[self.enabled]),
+            Case(enabled=False, test_subjects=[self.disabled]),
+            Case(enabled=True, test_subjects=[self.enabled, self.disabled]),
+        ]
+
+    def _create_penalizer(
+        self, case: Case
+    ) -> typing.Tuple[BatchedPenalizerOrchestrator, _BatchedPenalizer]:
+        orchestrator = BatchedPenalizerOrchestrator(
+            vocab_size=self.vocab_size,
+            batch=_BatchLike(reqs=[subject.to_req() for subject in case.test_subjects]),
+            device=self.device,
+            Penalizers={self.Penalizer},
+        )
+
+        return orchestrator, orchestrator.penalizers[self.Penalizer]
+
+    def test_is_required(self):
+        for case in self.test_cases:
+            with self.subTest(case=case):
+                _, penalizer = self._create_penalizer(case)
+                self.assertEqual(case.enabled, penalizer.is_required())
+
+    def test_prepare(self):
+        for case in self.test_cases:
+            with self.subTest(case=case):
+                orchestrator, penalizer = self._create_penalizer(case)
+                self.assertEqual(case.enabled, penalizer.is_prepared())
+
+                if case.enabled:
+                    for key, tensor in {
+                        key: torch.cat(
+                            tensors=[
+                                subject.steps[0].expected_tensors[key]
+                                for subject in case.test_subjects
+                            ],
+                        )
+                        for key in case.tensor_keys()
+                    }.items():
+                        torch.testing.assert_close(
+                            actual=getattr(penalizer, key),
+                            expected=tensor,
+                            msg=f"key={key}\nactual={getattr(penalizer, key)}\nexpected={tensor}",
+                        )
+
+                actual = orchestrator.apply(
+                    torch.ones(
+                        size=(len(case.test_subjects), self.vocab_size),
+                        dtype=torch.float32,
+                        device=self.device,
+                    )
+                )
+                expected = torch.cat(
+                    tensors=[
+                        subject.steps[0].expected_logits
+                        for subject in case.test_subjects
+                    ],
+                )
+                torch.testing.assert_close(
+                    actual=actual,
+                    expected=expected,
+                    msg=f"logits\nactual={actual}\nexpected={expected}",
+                )
+
+    def test_teardown(self):
+        for case in self.test_cases:
+            with self.subTest(case=case):
+                _, penalizer = self._create_penalizer(case)
+                penalizer.teardown()
+
+                for key in case.test_subjects[0].steps[0].expected_tensors.keys():
+                    self.assertIsNone(getattr(penalizer, key, None))
+
+    def test_filter(self):
+        for case in self.test_cases:
+            with self.subTest(case=case):
+                orchestrator, penalizer = self._create_penalizer(case)
+
+                indices_to_keep = [0]
+                orchestrator.filter(indices_to_keep=indices_to_keep)
+
+                filtered_subjects = [case.test_subjects[i] for i in indices_to_keep]
+
+                if penalizer.is_required():
+                    self.assertTrue(penalizer.is_prepared())
+                    for key, tensor in {
+                        key: torch.cat(
+                            tensors=[
+                                subject.steps[0].expected_tensors[key]
+                                for subject in filtered_subjects
+                            ],
+                        )
+                        for key in case.tensor_keys()
+                    }.items():
+                        torch.testing.assert_close(
+                            actual=getattr(penalizer, key),
+                            expected=tensor,
+                            msg=f"key={key}\nactual={getattr(penalizer, key)}\nexpected={tensor}",
+                        )
+
+                actual_logits = orchestrator.apply(
+                    torch.ones(
+                        size=(len(filtered_subjects), self.vocab_size),
+                        dtype=torch.float32,
+                        device=self.device,
+                    )
+                )
+                filtered_expected_logits = torch.cat(
+                    tensors=[
+                        subject.steps[0].expected_logits
+                        for subject in filtered_subjects
+                    ],
+                )
+                torch.testing.assert_close(
+                    actual=actual_logits,
+                    expected=filtered_expected_logits,
+                    msg=f"logits\nactual={actual_logits}\nexpected={filtered_expected_logits}",
+                )
+
+    def test_merge_enabled_with_disabled(self):
+        enabled_test_case = self.test_cases[0]
+        disabled_test_case = self.test_cases[1]
+
+        orchestrator, penalizer = self._create_penalizer(enabled_test_case)
+        theirs, _ = self._create_penalizer(disabled_test_case)
+
+        orchestrator.merge(theirs)
+
+        for key, tensor in {
+            key: torch.cat(
+                tensors=[
+                    enabled_test_case.test_subjects[0].steps[0].expected_tensors[key],
+                    disabled_test_case.test_subjects[0].steps[0].expected_tensors[key],
+                ],
+            )
+            for key in enabled_test_case.tensor_keys()
+        }.items():
+            torch.testing.assert_close(
+                actual=getattr(penalizer, key),
+                expected=tensor,
+                msg=f"key={key}\nactual={getattr(penalizer, key)}\nexpected={tensor}",
+            )
+
+    def test_cumulate_apply_repeat(self):
+        for case in self.test_cases:
+            with self.subTest(case=case):
+                orchestrator, penalizer = self._create_penalizer(case)
+
+                max_step = max(len(subject.steps) for subject in case.test_subjects)
+                for i in range(1, max_step):
+                    orchestrator.filter(
+                        indices_to_keep=[
+                            j
+                            for j, subject in enumerate(case.test_subjects)
+                            if i < len(subject.steps)
+                        ]
+                    )
+
+                    filtered_subjects = [
+                        subject
+                        for subject in case.test_subjects
+                        if i < len(subject.steps)
+                    ]
+
+                    inputs: typing.List[typing.List[int]] = []
+                    outputs: typing.List[typing.List[int]] = []
+                    for subject in filtered_subjects:
+                        step = subject.steps[i]
+                        if step.type == StepType.INPUT:
+                            inputs.append(step.token_ids)
+                            outputs.append([])
+                        else:
+                            inputs.append([])
+                            outputs.append(step.token_ids)
+
+                    if any(inputs):
+                        orchestrator.cumulate_input_tokens(inputs)
+
+                    if any(outputs):
+                        orchestrator.cumulate_output_tokens(outputs)
+
+                    if penalizer.is_required():
+                        self.assertTrue(penalizer.is_prepared())
+                        for key, tensor in {
+                            key: torch.cat(
+                                tensors=[
+                                    subject.steps[i].expected_tensors[key]
+                                    for subject in filtered_subjects
+                                ],
+                            )
+                            for key in case.tensor_keys()
+                        }.items():
+                            torch.testing.assert_close(
+                                actual=getattr(penalizer, key),
+                                expected=tensor,
+                                msg=f"key={key}\nactual={getattr(penalizer, key)}\nexpected={tensor}",
+                            )
+
+                    actual_logits = orchestrator.apply(
+                        torch.ones(
+                            size=(len(filtered_subjects), self.vocab_size),
+                            dtype=torch.float32,
+                            device=self.device,
+                        )
+                    )
+                    filtered_expected_logits = torch.cat(
+                        tensors=[
+                            subject.steps[i].expected_logits
+                            for subject in filtered_subjects
+                        ],
+                    )
+                    torch.testing.assert_close(
+                        actual=actual_logits,
+                        expected=filtered_expected_logits,
+                        msg=f"logits\nactual={actual_logits}\nexpected={filtered_expected_logits}",
+                    )

sglang/test/test_layernorm.py

@@ -0,0 +1,60 @@
+import itertools
+import unittest
+
+import torch
+
+from sglang.srt.layers.layernorm import RMSNorm
+
+
+class TestRMSNorm(unittest.TestCase):
+    DTYPES = [torch.half, torch.bfloat16]
+    NUM_TOKENS = [7, 83, 4096]
+    HIDDEN_SIZES = [768, 769, 770, 771, 5120, 5124, 5125, 5126, 8192, 8199]
+    ADD_RESIDUAL = [False, True]
+    SEEDS = [0]
+
+    @classmethod
+    def setUpClass(cls):
+        if not torch.cuda.is_available():
+            raise unittest.SkipTest("CUDA is not available")
+        torch.set_default_device("cuda")
+
+    def _run_rms_norm_test(self, num_tokens, hidden_size, add_residual, dtype, seed):
+        torch.manual_seed(seed)
+
+        layer = RMSNorm(hidden_size).to(dtype=dtype)
+        layer.weight.data.normal_(mean=1.0, std=0.1)
+        scale = 1 / (2 * hidden_size)
+        x = torch.randn(num_tokens, hidden_size, dtype=dtype) * scale
+        residual = torch.randn_like(x) * scale if add_residual else None
+
+        with torch.inference_mode():
+            ref_out = layer.forward_native(x, residual)
+            out = layer(x, residual)
+
+        if add_residual:
+            self.assertTrue(torch.allclose(out[0], ref_out[0], atol=1e-2, rtol=1e-2))
+            self.assertTrue(torch.allclose(out[1], ref_out[1], atol=1e-2, rtol=1e-2))
+        else:
+            self.assertTrue(torch.allclose(out, ref_out, atol=1e-2, rtol=1e-2))
+
+    def test_rms_norm(self):
+        for params in itertools.product(
+            self.NUM_TOKENS,
+            self.HIDDEN_SIZES,
+            self.ADD_RESIDUAL,
+            self.DTYPES,
+            self.SEEDS,
+        ):
+            with self.subTest(
+                num_tokens=params[0],
+                hidden_size=params[1],
+                add_residual=params[2],
+                dtype=params[3],
+                seed=params[4],
+            ):
+                self._run_rms_norm_test(*params)
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=2)

sglang/test/test_programs.py

@@ -149,7 +149,7 @@ def test_decode_json():
     assert isinstance(js_obj["population"], int)
 
 
-def test_expert_answer():
+def test_expert_answer(check_answer=True):
     @sgl.function
     def expert_answer(s, question):
         s += "Question: " + question + "\n"
@@ -167,7 +167,9 @@ def test_expert_answer():
         )
 
     ret = expert_answer.run(question="What is the capital of France?", temperature=0.1)
-    assert "paris" in ret.text().lower()
+
+    if check_answer:
+        assert "paris" in ret.text().lower(), f"Answer: {ret.text()}"
 
 
 def test_tool_use():