evalscope 0.10.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/benchmarks/drivelology/drivelology_binary_adapter.py

@@ -0,0 +1,170 @@
+# flake8: noqa: E501
+
+from typing import Any, Dict, List
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.messages import ChatMessageUser, Content, ContentText
+from evalscope.api.metric.scorer import AggScore, SampleScore, Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.logger import get_logger
+
+DESCRIPTION = (
+    'Drivelology, a unique linguistic phenomenon characterised as "nonsense with depth" - '
+    'utterances that are syntactically coherent yet pragmatically paradoxical, emotionally loaded, '
+    'or rhetorically subversive.'
+)
+
+PROMPT_TEMPLATE = """
+#Instruction#:
+Classify whether the given text is a Drivelology sample or not.
+
+#Definition#:
+- Drivelology: Statements that appear logically coherent but contain deeper, often paradoxical meanings.
+These challenge conventional interpretation by blending surface-level nonsense with underlying depth,
+often incorporating elements of humor, irony, or sarcasm, and requiring contextual understanding and
+emotional insight to unravel their true significance.
+- non-Drivelology: This includes pure nonsense (grammatically correct but semantically meaningless
+statements, such as "Colourless green ideas sleep furiously") and normal sentences, including quotes
+or proverbs, that convey clear or straightforward information without the layered complexity
+characteristic of Drivelology.
+
+#Output Format#:
+You should try your best to answer "Yes" if the given input text is Drivelology, otherwise specify "No".
+The answer you give MUST be \"Yes\" or \"No\"".
+
+#Input Text#: {text}
+#Your Answer#:
+""".strip()  # noqa: E501
+
+FEWSHOT_PROMPT_TEMPLATE = """
+#Instruction#:
+Classify whether the given text is a Drivelology sample or not.
+
+#Definition#:
+- Drivelology: Statements that appear logically coherent but contain deeper, often paradoxical meanings.
+These challenge conventional interpretation by blending surface-level nonsense with underlying depth,
+often incorporating elements of humor, irony, or sarcasm, and requiring contextual understanding and
+emotional insight to unravel their true significance.
+- non-Drivelology: This includes pure nonsense (grammatically correct but semantically meaningless
+statements, such as "Colourless green ideas sleep furiously") and normal sentences, including quotes
+or proverbs, that convey clear or straightforward information without the layered complexity
+characteristic of Drivelology.
+
+#Output Format#:
+You should try your best to answer "Yes" if the given input text is Drivelology, otherwise specify "No".
+The answer you give MUST be \"Yes\" or \"No\"".
+
+Here are some examples of how to solve similar problems:
+
+#Input Text#: Saw a book called "how to solve 50 percent of your problems" so I bought 2 books.
+#Your Answer#: Yes
+
+#Input Text#: Colourless green ideas sleep furiously.
+#Your Answer#: No
+
+#Input Text#: I went to a restaurant, and saw this guy was choking. I gotta save him. And then I realized he was just speaking French.
+#Your Answer#: Yes
+
+#Input Text#: Either it is or it isn't.
+#Your Answer#: No
+
+#Input Text#: {text}
+#Your Answer#:
+""".strip()  # noqa: E501
+
+logger = get_logger()
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='drivel_binary',
+        pretty_name='DrivelologyBinaryClassification',
+        tags=[Tags.YES_NO],
+        description=DESCRIPTION.strip(),
+        dataset_id='extraordinarylab/drivel-hub',
+        subset_list=['binary-classification'],
+        metric_list=['accuracy', 'precision', 'recall', 'f1_score', 'yes_ratio'],
+        aggregation='f1',
+        few_shot_num=0,
+        eval_split='test',
+        prompt_template='{question}',
+        few_shot_prompt_template='{question}'
+    )
+)
+class DrivelologyBinaryClassificationAdapter(DefaultDataAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.add_overall_metric = False
+        if self.few_shot_num not in [0, 4]:
+            logger.warning(f'For DrivelologyBinaryClassification, use 4-shot by default.')
+            self.few_shot_num = 4
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        if self.few_shot_num > 0:
+            prompt = FEWSHOT_PROMPT_TEMPLATE.format(text=record['text'])
+        else:
+            prompt = PROMPT_TEMPLATE.format(text=record['text'])
+        content_list: List[Content] = [ContentText(text=prompt)]
+        answer = 'YES' if str(record['label']) == 'drivelology' else 'NO'  # 'YES' or 'NO'
+        return Sample(input=[ChatMessageUser(content=content_list)], target=answer, metadata={
+            'answer': answer,
+        })
+
+    def match_score(self, original_prediction, filtered_prediction, reference, task_state) -> Score:
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+        # Check if the reference answer is in the filtered prediction
+        result = 1 if reference in filtered_prediction.strip().upper() else 0
+        score.value = {'acc': result}
+        return score
+
+    def aggregate_scores(self, sample_scores: List[SampleScore]) -> List[AggScore]:
+        """
+        Custom aggregation to compute accuracy, precision, recall, f1_score, and yes_ratio.
+        """
+
+        def compute_metrics(scores: List[SampleScore]):
+            tp = fp = tn = fn = 0
+            yes_count = 0
+            total_count = len(scores)
+
+            for ss in scores:
+                gt = ss.sample_metadata['answer'].strip().upper()
+                # Get prediction based on score
+                pred = gt if ss.score.main_value == 1 else ('NO' if gt == 'YES' else 'YES')
+                if pred == 'YES':
+                    yes_count += 1
+                if pred == 'YES' and gt == 'YES':
+                    tp += 1
+                elif pred == 'YES' and gt == 'NO':
+                    fp += 1
+                elif pred == 'NO' and gt == 'NO':
+                    tn += 1
+                elif pred == 'NO' and gt == 'YES':
+                    fn += 1
+
+            accuracy = (tp + tn) / total_count if total_count > 0 else 0.0
+            precision = tp / (tp + fp) if (tp + fp) > 0 else 0.0
+            recall = tp / (tp + fn) if (tp + fn) > 0 else 0.0
+            f1_score = (2 * precision * recall) / (precision + recall) if (precision + recall) > 0 else 0.0
+            yes_ratio = yes_count / total_count if total_count > 0 else 0.0
+
+            return {
+                'accuracy': accuracy,
+                'precision': precision,
+                'recall': recall,
+                'f1_score': f1_score,
+                'yes_ratio': yes_ratio
+            }
+
+        overall_metrics = compute_metrics(sample_scores)
+        agg_scores = []
+        for metric_name, value in overall_metrics.items():
+            agg_scores.append(AggScore(metric_name=metric_name, score=value, num=len(sample_scores), metadata={}))
+
+        return agg_scores

evalscope/benchmarks/drivelology/drivelology_multilabel_adapter.py

@@ -0,0 +1,254 @@
+import re
+from typing import Any, Dict, List
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.messages import ChatMessageUser, ContentText
+from evalscope.api.metric.scorer import AggScore, SampleScore, Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.logger import get_logger
+from evalscope.utils.multi_choices import parse_answers, prompt
+
+logger = get_logger()
+
+DESCRIPTION = (
+    'Drivelology, a unique linguistic phenomenon characterised as "nonsense with depth" - '
+    'utterances that are syntactically coherent yet pragmatically paradoxical, emotionally loaded, '
+    'or rhetorically subversive.'
+)
+
+MULTIPLE_ANSWER_TEMPLATE = r"""
+#Instruction#:
+Classify the given text into one or more of the following categories: inversion, wordplay, switchbait, paradox, and misdirection.
+
+#Definitions#:
+- inversion: This technique takes a well-known phrase, cliché, or social script and flips it on its head. The humour arises by reversing a familiar structure to creating a new, often satirical, meaning.
+- wordplay: This is the use of linguistic creativity, often by exploiting the phonetics or polysemy of words. It includes puns, double entendres, and similarities.
+- switchbait: This technique hinges on a specific phrase (the "bait") that has a culturally-embedded double meaning. The initial context is then suddenly replaced (the "switch") by a surprising second meaning. The humour is generated by this cynical or culturally-specific reinterpretation of the bait, rather than by derailing a narrative.
+- paradox: This relies on a statement that appears logically self-contradictory but contains a latent, often humorous or profound truth. The core of the technique is the clash of seemingly incompatible ideas.
+- misdirection: This technique leads the listener down an expected path before a final twist reveals a different, often more literal or absurd, ending.
+
+Answer the following multiple choice question where multiple answers may be correct.
+The entire content of your response should be of the following format: 'ANSWER: $LETTERS' (without quotes) where LETTERS is one or more of {letters}.
+
+{question}
+
+{choices}
+""".strip()  # noqa: E501
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='drivel_multilabel',
+        pretty_name='DrivelologyMultilabelClassification',
+        tags=[Tags.MULTIPLE_CHOICE],
+        description=DESCRIPTION.strip(),
+        dataset_id='extraordinarylab/drivel-hub',
+        subset_list=['multi-label-classification'],
+        metric_list=['f1_weighted', 'f1_micro', 'f1_macro', 'exact_match'],
+        aggregation='f1_weighted',
+        eval_split='test',
+        prompt_template='{question}',
+    )
+)
+class DrivelologyMultilabelClassificationAdapter(DefaultDataAdapter):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.categories = ['inversion', 'wordplay', 'switchbait', 'paradox', 'misdirection']
+        self.choices = {'A': 'inversion', 'B': 'wordplay', 'C': 'switchbait', 'D': 'paradox', 'E': 'misdirection'}
+        self.categories_to_letters = {v: k for k, v in self.choices.items()}
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        text: str = record['text']
+        label: List[str] = record['label']
+        question = f'Text to classify: {text}'
+        choices_list = [f'{key}. {value}' for key, value in self.choices.items()]
+        input_text = prompt(question=question, choices=choices_list, template=MULTIPLE_ANSWER_TEMPLATE)
+        content_list = [ContentText(text=input_text)]
+        target_letters = ''.join(
+            sorted([self.categories_to_letters[cat] for cat in label if cat in self.categories_to_letters])
+        )
+        metadata = {'text': text, 'label': label, 'target_letters': target_letters}
+        return Sample(
+            input=[ChatMessageUser(content=content_list)],
+            choices=choices_list,
+            target=target_letters,
+            metadata=metadata,
+        )
+
+    def extract_answer(self, prediction: str, task_state: TaskState) -> str:
+        pattern = r'ANSWER:\s*([A-E]+)'
+        match = re.search(pattern, prediction)
+        if match:
+            letters = match.group(1).strip().upper()
+            return ''.join(sorted(set(letters)))
+        else:
+            try:
+                answers = parse_answers(prediction)
+                return ''.join(sorted(list(answers)))
+            except Exception as e:
+                logger.warning(f'Could not extract answer from: {prediction}. Error: {e}')
+                return ''
+
+    def match_score(
+        self, original_prediction: str, filtered_prediction: str, reference: str, task_state: TaskState
+    ) -> Score:
+        """
+        Calculate the match score between the prediction and reference for multilabel classification.
+
+        Args:
+            original_prediction: The original model output
+            filtered_prediction: The extracted answer (letter format, e.g., "AC")
+            reference: The reference answer (letter format, e.g., "AC")
+            task_state: The current task state
+
+        Returns:
+            Score object with metrics
+        """
+        # Create a Score object as required by the API
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+
+        # Convert letter answers to category sets
+        pred_categories = set(self.choices.get(letter, '') for letter in filtered_prediction)
+        target_categories = set(self.choices.get(letter, '') for letter in reference)
+
+        # Remove empty strings (may be caused by invalid letters)
+        pred_categories = {cat for cat in pred_categories if cat}
+        target_categories = {cat for cat in target_categories if cat}
+
+        # Calculate TP (true positives), FP (false positives), and FN (false negatives)
+        tp = len(pred_categories & target_categories)  # intersection
+        fp = len(pred_categories - target_categories)  # in prediction but not in target
+        fn = len(target_categories - pred_categories)  # in target but not in prediction
+
+        # Calculate precision, recall and F1 score
+        precision = tp / (tp + fp) if (tp + fp) > 0 else 0.0
+        recall = tp / (tp + fn) if (tp + fn) > 0 else 0.0
+        f1 = 2 * precision * recall / (precision + recall) if (precision + recall) > 0 else 0.0
+
+        # Calculate exact match (1.0 if prediction exactly matches target)
+        exact_match = 1.0 if pred_categories == target_categories else 0.0
+
+        # Store category information in metadata for later aggregation
+        category_data = {}
+        for cat in self.categories:
+            in_pred = cat in pred_categories
+            in_target = cat in target_categories
+
+            category_data[cat] = {
+                'tp': 1 if in_pred and in_target else 0,
+                'fp': 1 if in_pred and not in_target else 0,
+                'fn': 1 if not in_pred and in_target else 0,
+                'support': 1 if in_target else 0
+            }
+
+        # Set simple numerical values in score.value as expected by the API
+        score.value = {'f1': f1, 'precision': precision, 'recall': recall, 'exact_match': exact_match}
+
+        # Store category data in metadata for aggregation
+        score.metadata = {'category_data': category_data}
+
+        return score
+
+    def aggregate_scores(self, sample_scores: List[SampleScore]) -> List[AggScore]:
+        """
+        Aggregate scores across all samples.
+        Computes weighted, macro, and micro F1 scores for multilabel classification.
+
+        Args:
+            sample_scores: List of sample scores
+
+        Returns:
+            List of aggregated scores
+        """
+        if not sample_scores:
+            return [
+                AggScore(metric_name='f1_weighted', score=0.0, num=0, metadata={}),
+                AggScore(metric_name='f1_micro', score=0.0, num=0, metadata={}),
+                AggScore(metric_name='f1_macro', score=0.0, num=0, metadata={}),
+                AggScore(metric_name='exact_match', score=0.0, num=0, metadata={})
+            ]
+
+        # Initialize category statistics
+        category_stats = {cat: {'tp': 0, 'fp': 0, 'fn': 0, 'support': 0} for cat in self.categories}
+        total_exact_matches = 0
+        num_samples = len(sample_scores)
+
+        # Aggregate statistics across all samples
+        for ss in sample_scores:
+            # Add exact match score to total
+            total_exact_matches += ss.score.value.get('exact_match', 0)
+
+            # Get category data from metadata
+            if 'category_data' in ss.score.metadata:
+                cat_data = ss.score.metadata['category_data']
+                for cat, stats in cat_data.items():
+                    if cat in self.categories:
+                        category_stats[cat]['tp'] += stats.get('tp', 0)
+                        category_stats[cat]['fp'] += stats.get('fp', 0)
+                        category_stats[cat]['fn'] += stats.get('fn', 0)
+                        category_stats[cat]['support'] += stats.get('support', 0)
+
+        # Calculate F1 scores for each category
+        category_f1 = {}
+        total_support = sum(stats['support'] for stats in category_stats.values())
+        f1_sum = 0.0
+
+        for cat, stats in category_stats.items():
+            tp = stats['tp']
+            fp = stats['fp']
+            fn = stats['fn']
+
+            precision = tp / (tp + fp) if (tp + fp) > 0 else 0.0
+            recall = tp / (tp + fn) if (tp + fn) > 0 else 0.0
+            f1 = 2 * precision * recall / (precision + recall) if (precision + recall) > 0 else 0.0
+
+            category_f1[cat] = f1
+            f1_sum += f1
+
+        # Calculate micro-average F1 (based on aggregate TP, FP, FN)
+        total_tp = sum(stats['tp'] for stats in category_stats.values())
+        total_fp = sum(stats['fp'] for stats in category_stats.values())
+        total_fn = sum(stats['fn'] for stats in category_stats.values())
+
+        micro_precision = total_tp / (total_tp + total_fp) if (total_tp + total_fp) > 0 else 0.0
+        micro_recall = total_tp / (total_tp + total_fn) if (total_tp + total_fn) > 0 else 0.0
+        f1_micro = 2 * micro_precision * micro_recall / (micro_precision + micro_recall) if (
+            micro_precision + micro_recall
+        ) > 0 else 0.0
+
+        # Calculate macro-average F1 (simple average of category F1 scores)
+        f1_macro = f1_sum / len(self.categories) if self.categories else 0.0
+
+        # Calculate weighted-average F1 (weighted by support)
+        f1_weighted = 0.0
+        if total_support > 0:
+            for cat, stats in category_stats.items():
+                cat_f1 = category_f1[cat]
+                weight = stats['support'] / total_support
+                f1_weighted += cat_f1 * weight
+
+        # Calculate accuracy (proportion of exact matches)
+        exact_match = total_exact_matches / num_samples
+
+        # Return list of aggregate scores
+        return [
+            AggScore(
+                metric_name='f1_weighted',
+                score=f1_weighted,
+                num=num_samples,
+                metadata={'category_f1': {
+                    cat: f1
+                    for cat, f1 in category_f1.items()
+                }}
+            ),
+            AggScore(metric_name='f1_micro', score=f1_micro, num=num_samples, metadata={}),
+            AggScore(metric_name='f1_macro', score=f1_macro, num=num_samples, metadata={}),
+            AggScore(metric_name='exact_match', score=exact_match, num=num_samples, metadata={})
+        ]

evalscope/benchmarks/drivelology/drivelology_selection_adapter.py

@@ -0,0 +1,49 @@
+from evalscope.api.benchmark import BenchmarkMeta, MultiChoiceAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+
+DESCRIPTION = (
+    'Drivelology, a unique linguistic phenomenon characterised as "nonsense with depth" - '
+    'utterances that are syntactically coherent yet pragmatically paradoxical, emotionally loaded, '
+    'or rhetorically subversive.'
+)
+
+PROMPT_TEMPLATE = r"""
+Tell me the best option in the following options which represents the underlying narrative of the text?
+The entire content of your response should be of the following format: 'ANSWER: $LETTER' (without quotes) where LETTER is one of {letters}.
+
+{question}
+
+{choices}
+""".strip()  # noqa: E501
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='drivel_selection',
+        pretty_name='DrivelologyNarrativeSelection',
+        tags=[Tags.MULTIPLE_CHOICE],
+        description=DESCRIPTION.strip(),
+        dataset_id='extraordinarylab/drivel-hub',
+        subset_list=['multiple-choice-english-easy', 'multiple-choice-english-hard'],
+        metric_list=['acc'],
+        few_shot_num=0,
+        train_split=None,
+        eval_split='test',
+        prompt_template=PROMPT_TEMPLATE,
+    )
+)
+class DrivelologyNarrativeSelectionAdapter(MultiChoiceAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.add_overall_metric = False
+
+    def record_to_sample(self, record) -> Sample:
+        return Sample(
+            input=record['text'],
+            choices=record['choices'],
+            target=record['answer'],
+            metadata={},
+        )

evalscope/benchmarks/drivelology/drivelology_writing_adapter.py

@@ -0,0 +1,218 @@
+import re
+from typing import Any, Dict, List
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.messages import ChatMessageUser, ContentText
+from evalscope.api.metric.scorer import AggScore, SampleScore, Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+DESCRIPTION = (
+    'Drivelology, a unique linguistic phenomenon characterised as "nonsense with depth" - '
+    'utterances that are syntactically coherent yet pragmatically paradoxical, emotionally loaded, '
+    'or rhetorically subversive.'
+)
+
+# Keep the original generation and evaluation templates
+NARRATIVE_GENERATION_TEMPLATE = """
+You need to first read and understand the text given. Generate a detailed description to illustrate the implicit narrative of the text.
+
+Please provide your response in English, with a clear and comprehensive explanation of the narrative.
+
+Text: {text}
+""".strip()  # noqa: E501
+
+NARRATIVE_EVALUATION_TEMPLATE = """
+Please act as an impartial judge and evaluate how accurately the candidate narrative matches the given reference narrative.
+Your evaluation should consider factors such as the relevance, accuracy, depth, and level of detail of the candidate narrative compared to the reference.
+
+Begin your evaluation by providing a short explanation in English. Be as objective as possible.
+
+After providing your explanation, you must rate the match on a Likert scale from 1 to 5, where:
+1 = Very poor match
+2 = Poor match
+3 = Moderate match
+4 = Good match
+5 = Excellent match
+
+Please format your rating strictly as: "Rating: [[X]]" where X is a whole number from 1 to 5.
+
+[Candidate Narrative]
+{candidate}
+
+[Reference Narrative]
+{reference}
+""".strip()  # noqa: E501
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='drivel_writing',
+        pretty_name='DrivelologyNarrativeWriting',
+        tags=[Tags.KNOWLEDGE, Tags.REASONING],
+        description=DESCRIPTION.strip(),
+        dataset_id='extraordinarylab/drivel-hub',
+        subset_list=['narrative-writing-english'],
+        metric_list={
+            'bert_score': {
+                'model_id_or_path': 'AI-ModelScope/roberta-large',
+                'model_type': 'roberta-large'
+            },
+            'gpt_score': {}
+        },
+        few_shot_num=0,
+        train_split=None,
+        eval_split='test',
+        prompt_template=NARRATIVE_GENERATION_TEMPLATE
+    )
+)
+class DrivelologyNarrativeWritingAdapter(DefaultDataAdapter):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._use_llm_judge = True  # Use LLM as a judge by default
+        self.use_batch_scoring = True  # Enable batch scoring
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        """
+        Convert a data record to a Sample object.
+        """
+        text = record['text']
+        reference_narrative = record['narrative']
+
+        # Format the generation prompt with the text
+        input_prompt = NARRATIVE_GENERATION_TEMPLATE.format(text=text)
+
+        # Create content list for the input
+        content_list = [ContentText(text=input_prompt)]
+
+        return Sample(
+            input=[ChatMessageUser(content=content_list)],
+            target=reference_narrative,
+            metadata={
+                'text': text,
+                'reference_narrative': reference_narrative
+            }
+        )
+
+    def batch_match_score(self, original_predictions, filtered_predictions, references, task_states):
+        """
+        Batch calculate the match scores using BERTScore.
+        """
+        from evalscope.metrics.metric import BertScore
+
+        score_args = self.metric_list.get('bert_score', {})
+        bert_scorer = BertScore(**score_args)
+        bert_score_f1 = bert_scorer.apply(filtered_predictions, references)
+        scores = []
+        for i in range(len(original_predictions)):
+            score = Score(
+                extracted_prediction=filtered_predictions[i],
+                prediction=original_predictions[i],
+                value={'bert_score': bert_score_f1[i]}
+            )
+            scores.append(score)
+        return scores
+
+    def llm_match_score(
+        self,
+        original_prediction: str,
+        filtered_prediction: str,
+        reference: str,
+        task_state: TaskState,
+    ) -> Score:
+        """
+        Calculate the match score using LLM judge and BERTScore.
+        """
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+
+        # Initialize score value dictionary
+        score.value = {}
+
+        # Use LLM judge to evaluate narrative quality
+        eval_prompt = NARRATIVE_EVALUATION_TEMPLATE.format(candidate=filtered_prediction, reference=reference)
+
+        judge_response = self.llm_judge.judge(eval_prompt)
+        logger.info(f'LLM judge response received (first 100 chars): {judge_response[:100]}...')
+
+        # Extract rating using regex pattern
+        match = re.search(r'Rating:\s*\[\[([1-5])\]\]', judge_response)
+        if match:
+            rating = int(match.group(1))
+            gpt_score = (rating - 1) / 4.0  # Normalize to 0-1 scale
+            logger.info(f'Rating extracted: {rating}/5 -> {gpt_score}')
+        else:
+            # Try alternative pattern
+            alt_match = re.search(r'(\[\[|\[)([1-5])(\]\]|\])', judge_response)
+            if alt_match:
+                rating = int(alt_match.group(2))
+                gpt_score = (rating - 1) / 4.0
+                logger.info(f'Rating extracted (alt pattern): {rating}/5 -> {gpt_score}')
+            else:
+                # Last resort: standalone digit
+                number_match = re.search(r'(?<!\d)[1-5](?!\d)', judge_response)
+                if number_match:
+                    rating = int(number_match.group(0))
+                    gpt_score = (rating - 1) / 4.0
+                    logger.info(f'Rating extracted (fallback): {rating}/5 -> {gpt_score}')
+                else:
+                    gpt_score = 0.0
+                    logger.warning('No rating found in response, using default 0.0')
+
+        score.value['gpt_score'] = gpt_score
+        score.explanation = f'LLM judge rating: {gpt_score:.2f}'
+
+        score.metadata = {
+            'judge_response': judge_response[:300],
+            'model': getattr(self.llm_judge, 'model_id', 'unknown')
+        }
+
+        score.main_score_name = 'gpt_score'
+        return score
+
+    def aggregate_scores(self, sample_scores: List[SampleScore]) -> List[AggScore]:
+        """
+        Aggregate scores across all samples.
+        """
+        if not sample_scores:
+            return [
+                AggScore(metric_name='gpt_score', score=0.0, num=0, metadata={}),
+                AggScore(metric_name='bert_score', score=0.0, num=0, metadata={})
+            ]
+
+        # Extract scores
+        gpt_scores = [ss.score.value.get('gpt_score', 0.0) for ss in sample_scores]
+        bert_scores = [ss.score.value.get('bert_score', 0.0) for ss in sample_scores]
+
+        # Calculate averages
+        avg_gpt_score = sum(gpt_scores) / len(gpt_scores) if gpt_scores else 0.0
+        avg_bert_score = sum(bert_scores) / len(bert_scores) if bert_scores else 0.0
+
+        return [
+            AggScore(
+                metric_name='gpt_score',
+                score=avg_gpt_score,
+                num=len(sample_scores),
+                metadata={
+                    'min_score': min(gpt_scores),
+                    'max_score': max(gpt_scores)
+                }
+            ),
+            AggScore(
+                metric_name='bert_score',
+                score=avg_bert_score,
+                num=len(sample_scores),
+                metadata={
+                    'min_score': min(bert_scores),
+                    'max_score': max(bert_scores)
+                }
+            )
+        ]