evalscope 1.0.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/benchmarks/drivelology/drivelology_multilabel_adapter.py

@@ -0,0 +1,254 @@
+import re
+from typing import Any, Dict, List
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.messages import ChatMessageUser, ContentText
+from evalscope.api.metric.scorer import AggScore, SampleScore, Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.logger import get_logger
+from evalscope.utils.multi_choices import parse_answers, prompt
+
+logger = get_logger()
+
+DESCRIPTION = (
+    'Drivelology, a unique linguistic phenomenon characterised as "nonsense with depth" - '
+    'utterances that are syntactically coherent yet pragmatically paradoxical, emotionally loaded, '
+    'or rhetorically subversive.'
+)
+
+MULTIPLE_ANSWER_TEMPLATE = r"""
+#Instruction#:
+Classify the given text into one or more of the following categories: inversion, wordplay, switchbait, paradox, and misdirection.
+
+#Definitions#:
+- inversion: This technique takes a well-known phrase, cliché, or social script and flips it on its head. The humour arises by reversing a familiar structure to creating a new, often satirical, meaning.
+- wordplay: This is the use of linguistic creativity, often by exploiting the phonetics or polysemy of words. It includes puns, double entendres, and similarities.
+- switchbait: This technique hinges on a specific phrase (the "bait") that has a culturally-embedded double meaning. The initial context is then suddenly replaced (the "switch") by a surprising second meaning. The humour is generated by this cynical or culturally-specific reinterpretation of the bait, rather than by derailing a narrative.
+- paradox: This relies on a statement that appears logically self-contradictory but contains a latent, often humorous or profound truth. The core of the technique is the clash of seemingly incompatible ideas.
+- misdirection: This technique leads the listener down an expected path before a final twist reveals a different, often more literal or absurd, ending.
+
+Answer the following multiple choice question where multiple answers may be correct.
+The entire content of your response should be of the following format: 'ANSWER: $LETTERS' (without quotes) where LETTERS is one or more of {letters}.
+
+{question}
+
+{choices}
+""".strip()  # noqa: E501
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='drivel_multilabel',
+        pretty_name='DrivelologyMultilabelClassification',
+        tags=[Tags.MULTIPLE_CHOICE],
+        description=DESCRIPTION.strip(),
+        dataset_id='extraordinarylab/drivel-hub',
+        subset_list=['multi-label-classification'],
+        metric_list=['f1_weighted', 'f1_micro', 'f1_macro', 'exact_match'],
+        aggregation='f1_weighted',
+        eval_split='test',
+        prompt_template='{question}',
+    )
+)
+class DrivelologyMultilabelClassificationAdapter(DefaultDataAdapter):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.categories = ['inversion', 'wordplay', 'switchbait', 'paradox', 'misdirection']
+        self.choices = {'A': 'inversion', 'B': 'wordplay', 'C': 'switchbait', 'D': 'paradox', 'E': 'misdirection'}
+        self.categories_to_letters = {v: k for k, v in self.choices.items()}
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        text: str = record['text']
+        label: List[str] = record['label']
+        question = f'Text to classify: {text}'
+        choices_list = [f'{key}. {value}' for key, value in self.choices.items()]
+        input_text = prompt(question=question, choices=choices_list, template=MULTIPLE_ANSWER_TEMPLATE)
+        content_list = [ContentText(text=input_text)]
+        target_letters = ''.join(
+            sorted([self.categories_to_letters[cat] for cat in label if cat in self.categories_to_letters])
+        )
+        metadata = {'text': text, 'label': label, 'target_letters': target_letters}
+        return Sample(
+            input=[ChatMessageUser(content=content_list)],
+            choices=choices_list,
+            target=target_letters,
+            metadata=metadata,
+        )
+
+    def extract_answer(self, prediction: str, task_state: TaskState) -> str:
+        pattern = r'ANSWER:\s*([A-E]+)'
+        match = re.search(pattern, prediction)
+        if match:
+            letters = match.group(1).strip().upper()
+            return ''.join(sorted(set(letters)))
+        else:
+            try:
+                answers = parse_answers(prediction)
+                return ''.join(sorted(list(answers)))
+            except Exception as e:
+                logger.warning(f'Could not extract answer from: {prediction}. Error: {e}')
+                return ''
+
+    def match_score(
+        self, original_prediction: str, filtered_prediction: str, reference: str, task_state: TaskState
+    ) -> Score:
+        """
+        Calculate the match score between the prediction and reference for multilabel classification.
+
+        Args:
+            original_prediction: The original model output
+            filtered_prediction: The extracted answer (letter format, e.g., "AC")
+            reference: The reference answer (letter format, e.g., "AC")
+            task_state: The current task state
+
+        Returns:
+            Score object with metrics
+        """
+        # Create a Score object as required by the API
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+
+        # Convert letter answers to category sets
+        pred_categories = set(self.choices.get(letter, '') for letter in filtered_prediction)
+        target_categories = set(self.choices.get(letter, '') for letter in reference)
+
+        # Remove empty strings (may be caused by invalid letters)
+        pred_categories = {cat for cat in pred_categories if cat}
+        target_categories = {cat for cat in target_categories if cat}
+
+        # Calculate TP (true positives), FP (false positives), and FN (false negatives)
+        tp = len(pred_categories & target_categories)  # intersection
+        fp = len(pred_categories - target_categories)  # in prediction but not in target
+        fn = len(target_categories - pred_categories)  # in target but not in prediction
+
+        # Calculate precision, recall and F1 score
+        precision = tp / (tp + fp) if (tp + fp) > 0 else 0.0
+        recall = tp / (tp + fn) if (tp + fn) > 0 else 0.0
+        f1 = 2 * precision * recall / (precision + recall) if (precision + recall) > 0 else 0.0
+
+        # Calculate exact match (1.0 if prediction exactly matches target)
+        exact_match = 1.0 if pred_categories == target_categories else 0.0
+
+        # Store category information in metadata for later aggregation
+        category_data = {}
+        for cat in self.categories:
+            in_pred = cat in pred_categories
+            in_target = cat in target_categories
+
+            category_data[cat] = {
+                'tp': 1 if in_pred and in_target else 0,
+                'fp': 1 if in_pred and not in_target else 0,
+                'fn': 1 if not in_pred and in_target else 0,
+                'support': 1 if in_target else 0
+            }
+
+        # Set simple numerical values in score.value as expected by the API
+        score.value = {'f1': f1, 'precision': precision, 'recall': recall, 'exact_match': exact_match}
+
+        # Store category data in metadata for aggregation
+        score.metadata = {'category_data': category_data}
+
+        return score
+
+    def aggregate_scores(self, sample_scores: List[SampleScore]) -> List[AggScore]:
+        """
+        Aggregate scores across all samples.
+        Computes weighted, macro, and micro F1 scores for multilabel classification.
+
+        Args:
+            sample_scores: List of sample scores
+
+        Returns:
+            List of aggregated scores
+        """
+        if not sample_scores:
+            return [
+                AggScore(metric_name='f1_weighted', score=0.0, num=0, metadata={}),
+                AggScore(metric_name='f1_micro', score=0.0, num=0, metadata={}),
+                AggScore(metric_name='f1_macro', score=0.0, num=0, metadata={}),
+                AggScore(metric_name='exact_match', score=0.0, num=0, metadata={})
+            ]
+
+        # Initialize category statistics
+        category_stats = {cat: {'tp': 0, 'fp': 0, 'fn': 0, 'support': 0} for cat in self.categories}
+        total_exact_matches = 0
+        num_samples = len(sample_scores)
+
+        # Aggregate statistics across all samples
+        for ss in sample_scores:
+            # Add exact match score to total
+            total_exact_matches += ss.score.value.get('exact_match', 0)
+
+            # Get category data from metadata
+            if 'category_data' in ss.score.metadata:
+                cat_data = ss.score.metadata['category_data']
+                for cat, stats in cat_data.items():
+                    if cat in self.categories:
+                        category_stats[cat]['tp'] += stats.get('tp', 0)
+                        category_stats[cat]['fp'] += stats.get('fp', 0)
+                        category_stats[cat]['fn'] += stats.get('fn', 0)
+                        category_stats[cat]['support'] += stats.get('support', 0)
+
+        # Calculate F1 scores for each category
+        category_f1 = {}
+        total_support = sum(stats['support'] for stats in category_stats.values())
+        f1_sum = 0.0
+
+        for cat, stats in category_stats.items():
+            tp = stats['tp']
+            fp = stats['fp']
+            fn = stats['fn']
+
+            precision = tp / (tp + fp) if (tp + fp) > 0 else 0.0
+            recall = tp / (tp + fn) if (tp + fn) > 0 else 0.0
+            f1 = 2 * precision * recall / (precision + recall) if (precision + recall) > 0 else 0.0
+
+            category_f1[cat] = f1
+            f1_sum += f1
+
+        # Calculate micro-average F1 (based on aggregate TP, FP, FN)
+        total_tp = sum(stats['tp'] for stats in category_stats.values())
+        total_fp = sum(stats['fp'] for stats in category_stats.values())
+        total_fn = sum(stats['fn'] for stats in category_stats.values())
+
+        micro_precision = total_tp / (total_tp + total_fp) if (total_tp + total_fp) > 0 else 0.0
+        micro_recall = total_tp / (total_tp + total_fn) if (total_tp + total_fn) > 0 else 0.0
+        f1_micro = 2 * micro_precision * micro_recall / (micro_precision + micro_recall) if (
+            micro_precision + micro_recall
+        ) > 0 else 0.0
+
+        # Calculate macro-average F1 (simple average of category F1 scores)
+        f1_macro = f1_sum / len(self.categories) if self.categories else 0.0
+
+        # Calculate weighted-average F1 (weighted by support)
+        f1_weighted = 0.0
+        if total_support > 0:
+            for cat, stats in category_stats.items():
+                cat_f1 = category_f1[cat]
+                weight = stats['support'] / total_support
+                f1_weighted += cat_f1 * weight
+
+        # Calculate accuracy (proportion of exact matches)
+        exact_match = total_exact_matches / num_samples
+
+        # Return list of aggregate scores
+        return [
+            AggScore(
+                metric_name='f1_weighted',
+                score=f1_weighted,
+                num=num_samples,
+                metadata={'category_f1': {
+                    cat: f1
+                    for cat, f1 in category_f1.items()
+                }}
+            ),
+            AggScore(metric_name='f1_micro', score=f1_micro, num=num_samples, metadata={}),
+            AggScore(metric_name='f1_macro', score=f1_macro, num=num_samples, metadata={}),
+            AggScore(metric_name='exact_match', score=exact_match, num=num_samples, metadata={})
+        ]

evalscope/benchmarks/drivelology/drivelology_selection_adapter.py

@@ -0,0 +1,49 @@
+from evalscope.api.benchmark import BenchmarkMeta, MultiChoiceAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+
+DESCRIPTION = (
+    'Drivelology, a unique linguistic phenomenon characterised as "nonsense with depth" - '
+    'utterances that are syntactically coherent yet pragmatically paradoxical, emotionally loaded, '
+    'or rhetorically subversive.'
+)
+
+PROMPT_TEMPLATE = r"""
+Tell me the best option in the following options which represents the underlying narrative of the text?
+The entire content of your response should be of the following format: 'ANSWER: $LETTER' (without quotes) where LETTER is one of {letters}.
+
+{question}
+
+{choices}
+""".strip()  # noqa: E501
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='drivel_selection',
+        pretty_name='DrivelologyNarrativeSelection',
+        tags=[Tags.MULTIPLE_CHOICE],
+        description=DESCRIPTION.strip(),
+        dataset_id='extraordinarylab/drivel-hub',
+        subset_list=['multiple-choice-english-easy', 'multiple-choice-english-hard'],
+        metric_list=['acc'],
+        few_shot_num=0,
+        train_split=None,
+        eval_split='test',
+        prompt_template=PROMPT_TEMPLATE,
+    )
+)
+class DrivelologyNarrativeSelectionAdapter(MultiChoiceAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.add_overall_metric = False
+
+    def record_to_sample(self, record) -> Sample:
+        return Sample(
+            input=record['text'],
+            choices=record['choices'],
+            target=record['answer'],
+            metadata={},
+        )

evalscope/benchmarks/drivelology/drivelology_writing_adapter.py

@@ -0,0 +1,218 @@
+import re
+from typing import Any, Dict, List
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.messages import ChatMessageUser, ContentText
+from evalscope.api.metric.scorer import AggScore, SampleScore, Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+DESCRIPTION = (
+    'Drivelology, a unique linguistic phenomenon characterised as "nonsense with depth" - '
+    'utterances that are syntactically coherent yet pragmatically paradoxical, emotionally loaded, '
+    'or rhetorically subversive.'
+)
+
+# Keep the original generation and evaluation templates
+NARRATIVE_GENERATION_TEMPLATE = """
+You need to first read and understand the text given. Generate a detailed description to illustrate the implicit narrative of the text.
+
+Please provide your response in English, with a clear and comprehensive explanation of the narrative.
+
+Text: {text}
+""".strip()  # noqa: E501
+
+NARRATIVE_EVALUATION_TEMPLATE = """
+Please act as an impartial judge and evaluate how accurately the candidate narrative matches the given reference narrative.
+Your evaluation should consider factors such as the relevance, accuracy, depth, and level of detail of the candidate narrative compared to the reference.
+
+Begin your evaluation by providing a short explanation in English. Be as objective as possible.
+
+After providing your explanation, you must rate the match on a Likert scale from 1 to 5, where:
+1 = Very poor match
+2 = Poor match
+3 = Moderate match
+4 = Good match
+5 = Excellent match
+
+Please format your rating strictly as: "Rating: [[X]]" where X is a whole number from 1 to 5.
+
+[Candidate Narrative]
+{candidate}
+
+[Reference Narrative]
+{reference}
+""".strip()  # noqa: E501
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='drivel_writing',
+        pretty_name='DrivelologyNarrativeWriting',
+        tags=[Tags.KNOWLEDGE, Tags.REASONING],
+        description=DESCRIPTION.strip(),
+        dataset_id='extraordinarylab/drivel-hub',
+        subset_list=['narrative-writing-english'],
+        metric_list={
+            'bert_score': {
+                'model_id_or_path': 'AI-ModelScope/roberta-large',
+                'model_type': 'roberta-large'
+            },
+            'gpt_score': {}
+        },
+        few_shot_num=0,
+        train_split=None,
+        eval_split='test',
+        prompt_template=NARRATIVE_GENERATION_TEMPLATE
+    )
+)
+class DrivelologyNarrativeWritingAdapter(DefaultDataAdapter):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._use_llm_judge = True  # Use LLM as a judge by default
+        self.use_batch_scoring = True  # Enable batch scoring
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        """
+        Convert a data record to a Sample object.
+        """
+        text = record['text']
+        reference_narrative = record['narrative']
+
+        # Format the generation prompt with the text
+        input_prompt = NARRATIVE_GENERATION_TEMPLATE.format(text=text)
+
+        # Create content list for the input
+        content_list = [ContentText(text=input_prompt)]
+
+        return Sample(
+            input=[ChatMessageUser(content=content_list)],
+            target=reference_narrative,
+            metadata={
+                'text': text,
+                'reference_narrative': reference_narrative
+            }
+        )
+
+    def batch_match_score(self, original_predictions, filtered_predictions, references, task_states):
+        """
+        Batch calculate the match scores using BERTScore.
+        """
+        from evalscope.metrics.metric import BertScore
+
+        score_args = self.metric_list.get('bert_score', {})
+        bert_scorer = BertScore(**score_args)
+        bert_score_f1 = bert_scorer.apply(filtered_predictions, references)
+        scores = []
+        for i in range(len(original_predictions)):
+            score = Score(
+                extracted_prediction=filtered_predictions[i],
+                prediction=original_predictions[i],
+                value={'bert_score': bert_score_f1[i]}
+            )
+            scores.append(score)
+        return scores
+
+    def llm_match_score(
+        self,
+        original_prediction: str,
+        filtered_prediction: str,
+        reference: str,
+        task_state: TaskState,
+    ) -> Score:
+        """
+        Calculate the match score using LLM judge and BERTScore.
+        """
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+
+        # Initialize score value dictionary
+        score.value = {}
+
+        # Use LLM judge to evaluate narrative quality
+        eval_prompt = NARRATIVE_EVALUATION_TEMPLATE.format(candidate=filtered_prediction, reference=reference)
+
+        judge_response = self.llm_judge.judge(eval_prompt)
+        logger.info(f'LLM judge response received (first 100 chars): {judge_response[:100]}...')
+
+        # Extract rating using regex pattern
+        match = re.search(r'Rating:\s*\[\[([1-5])\]\]', judge_response)
+        if match:
+            rating = int(match.group(1))
+            gpt_score = (rating - 1) / 4.0  # Normalize to 0-1 scale
+            logger.info(f'Rating extracted: {rating}/5 -> {gpt_score}')
+        else:
+            # Try alternative pattern
+            alt_match = re.search(r'(\[\[|\[)([1-5])(\]\]|\])', judge_response)
+            if alt_match:
+                rating = int(alt_match.group(2))
+                gpt_score = (rating - 1) / 4.0
+                logger.info(f'Rating extracted (alt pattern): {rating}/5 -> {gpt_score}')
+            else:
+                # Last resort: standalone digit
+                number_match = re.search(r'(?<!\d)[1-5](?!\d)', judge_response)
+                if number_match:
+                    rating = int(number_match.group(0))
+                    gpt_score = (rating - 1) / 4.0
+                    logger.info(f'Rating extracted (fallback): {rating}/5 -> {gpt_score}')
+                else:
+                    gpt_score = 0.0
+                    logger.warning('No rating found in response, using default 0.0')
+
+        score.value['gpt_score'] = gpt_score
+        score.explanation = f'LLM judge rating: {gpt_score:.2f}'
+
+        score.metadata = {
+            'judge_response': judge_response[:300],
+            'model': getattr(self.llm_judge, 'model_id', 'unknown')
+        }
+
+        score.main_score_name = 'gpt_score'
+        return score
+
+    def aggregate_scores(self, sample_scores: List[SampleScore]) -> List[AggScore]:
+        """
+        Aggregate scores across all samples.
+        """
+        if not sample_scores:
+            return [
+                AggScore(metric_name='gpt_score', score=0.0, num=0, metadata={}),
+                AggScore(metric_name='bert_score', score=0.0, num=0, metadata={})
+            ]
+
+        # Extract scores
+        gpt_scores = [ss.score.value.get('gpt_score', 0.0) for ss in sample_scores]
+        bert_scores = [ss.score.value.get('bert_score', 0.0) for ss in sample_scores]
+
+        # Calculate averages
+        avg_gpt_score = sum(gpt_scores) / len(gpt_scores) if gpt_scores else 0.0
+        avg_bert_score = sum(bert_scores) / len(bert_scores) if bert_scores else 0.0
+
+        return [
+            AggScore(
+                metric_name='gpt_score',
+                score=avg_gpt_score,
+                num=len(sample_scores),
+                metadata={
+                    'min_score': min(gpt_scores),
+                    'max_score': max(gpt_scores)
+                }
+            ),
+            AggScore(
+                metric_name='bert_score',
+                score=avg_bert_score,
+                num=len(sample_scores),
+                metadata={
+                    'min_score': min(bert_scores),
+                    'max_score': max(bert_scores)
+                }
+            )
+        ]

evalscope/benchmarks/drop/drop_adapter.py

@@ -41,7 +41,7 @@ Answer:  43
         description=
         'The DROP (Discrete Reasoning Over Paragraphs) benchmark is designed to evaluate the reading comprehension and reasoning capabilities of AI models. It includes a variety of tasks that require models to read passages and answer questions based on the content.',  # noqa: E501
         dataset_id='AI-ModelScope/DROP',
-        metric_list=['acc'],
+        metric_list=['em', 'f1'],
         few_shot_num=3,
         train_split=None,
         eval_split='validation',
@@ -54,11 +54,9 @@ class DROPAdapter(DefaultDataAdapter):
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
-        if self.few_shot_num != 0:
+        if self.few_shot_num != 0 and self.few_shot_num != 3:
             self.few_shot_num = 3
             logger.info(f'Few shot num is set to {self.few_shot_num} for DROP dataset by system.')
-        else:
-            self.few_shot_num = 0
 
     def record_to_sample(self, record: Dict[str, Any]) -> Sample:
         """
@@ -70,9 +68,10 @@ class DROPAdapter(DefaultDataAdapter):
         Returns:
             Sample: Sample object with input, target, and metadata.
         """
+        from .utils import _get_gold_answers
 
         # Parse gold answers
-        gold_answers = self._get_gold_answers(record)
+        gold_answers = _get_gold_answers(record)
 
         return Sample(
             input=record['question'],
@@ -102,33 +101,6 @@ class DROPAdapter(DefaultDataAdapter):
             query=query,
         )
 
-    def _get_gold_answers(self, input_d: dict) -> List[str]:
-        """
-        Parse the raw input labels (gold).
-        """
-
-        def _flatten_validated_answers(validated_answers):
-            """Flattens a dict of lists of validated answers."""
-            valid_answers = []
-            for i in range(len(validated_answers['number'])):
-                valid_answers.append({
-                    'number': validated_answers['number'][i],
-                    'date': validated_answers['date'][i],
-                    'spans': validated_answers['spans'][i],
-                })
-            return valid_answers
-
-        answers = []
-        answers_set = set()
-        candidates = [input_d['answer']] + _flatten_validated_answers(input_d['validated_answers'])
-        for candidate in candidates:
-            answer = DROPAdapter.parse_answer(candidate)
-            if answer in answers_set:
-                continue
-            answers_set.add(answer)
-            answers.append(answer)
-        return answers
-
     def extract_answer(self, prediction: str, task_state: TaskState):
         """
         Extract the answer from the model prediction.
@@ -147,7 +119,9 @@ class DROPAdapter(DefaultDataAdapter):
         """
         Calculate accuracy score by matching prediction with reference answers.
         """
-        from .utils import _answer_to_bags
+        import numpy as np
+
+        from .utils import _align_bags, _answer_to_bags
 
         score = Score(
             extracted_prediction=filtered_prediction,
@@ -155,6 +129,7 @@ class DROPAdapter(DefaultDataAdapter):
         )
 
         max_em = 0
+        max_f1 = 0
         reference = ast.literal_eval(reference) if isinstance(reference, str) else reference
         for gold_answer in reference:
             # Convert the answers to bags of answers
@@ -165,20 +140,16 @@ class DROPAdapter(DefaultDataAdapter):
                 exact_match = 1.0
             else:
                 exact_match = 0.0
+
+            f1_per_bag = _align_bags(predicted_bags[1], gold_bags[1])
+            f1_score = np.mean(f1_per_bag)
+            f1_score = round(f1_score, 2)
             # Check if the answer is empty
             if gold_answer[0].strip():
                 max_em = max(max_em, exact_match)
+                max_f1 = max(max_f1, f1_score)
 
-        score.value = {'acc': max_em}
-        score.main_score_name = 'acc'
+        score.value = {'em': max_em, 'f1': max_f1}
+        score.main_score_name = 'f1'
 
         return score
-
-    @staticmethod
-    def parse_answer(answer):
-        # NOTE: Everything is returned as a tuple for uniformity and hashability.
-        if answer['number'] != '':
-            return (str(answer['number']), )
-        if answer['spans'] != []:
-            return tuple(answer['spans'])
-        return (' '.join([answer['date']['day'], answer['date']['month'], answer['date']['year']]).strip(), )