evalscope 0.7.2__py3-none-any.whl → 0.8.1__py3-none-any.whl

evalscope/benchmarks/cmmlu/cmmlu_adapter.py

@@ -1,11 +1,13 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-import os
 import csv
+import os
+
 from evalscope.benchmarks.data_adapter import DataAdapter
 from evalscope.metrics.metrics import exact_match, weighted_mean
 from evalscope.utils import ResponseParser, normalize_score
 from evalscope.utils.logger import get_logger
+
 # flake8: noqa
 
 logger = get_logger()
@@ -13,143 +15,89 @@ logger = get_logger()
 DATASET_ID = 'modelscope/cmmlu'
 
 SUBSET_LIST = [
-    "agronomy",
-    "anatomy",
-    "ancient_chinese",
-    "arts",
-    "astronomy",
-    "business_ethics",
-    "chinese_civil_service_exam",
-    "chinese_driving_rule",
-    "chinese_food_culture",
-    "chinese_foreign_policy",
-    "chinese_history",
-    "chinese_literature",
-    "chinese_teacher_qualification",
-    "college_actuarial_science",
-    "college_education",
-    "college_engineering_hydrology",
-    "college_law",
-    "college_mathematics",
-    "college_medical_statistics",
-    "clinical_knowledge",
-    "college_medicine",
-    "computer_science",
-    "computer_security",
-    "conceptual_physics",
-    "construction_project_management",
-    "economics",
-    "education",
-    "elementary_chinese",
-    "elementary_commonsense",
-    "elementary_information_and_technology",
-    "electrical_engineering",
-    "elementary_mathematics",
-    "ethnology",
-    "food_science",
-    "genetics",
-    "global_facts",
-    "high_school_biology",
-    "high_school_chemistry",
-    "high_school_geography",
-    "high_school_mathematics",
-    "high_school_physics",
-    "high_school_politics",
-    "human_sexuality",
-    "international_law",
-    "journalism",
-    "jurisprudence",
-    "legal_and_moral_basis",
-    "logical",
-    "machine_learning",
-    "management",
-    "marketing",
-    "marxist_theory",
-    "modern_chinese",
-    "nutrition",
-    "philosophy",
-    "professional_accounting",
-    "professional_law",
-    "professional_medicine",
-    "professional_psychology",
-    "public_relations",
-    "security_study",
-    "sociology",
-    "sports_science",
-    "traditional_chinese_medicine",
-    "virology",
-    "world_history",
-    "world_religions"
+    'agronomy', 'anatomy', 'ancient_chinese', 'arts', 'astronomy', 'business_ethics', 'chinese_civil_service_exam',
+    'chinese_driving_rule', 'chinese_food_culture', 'chinese_foreign_policy', 'chinese_history', 'chinese_literature',
+    'chinese_teacher_qualification', 'college_actuarial_science', 'college_education', 'college_engineering_hydrology',
+    'college_law', 'college_mathematics', 'college_medical_statistics', 'clinical_knowledge', 'college_medicine',
+    'computer_science', 'computer_security', 'conceptual_physics', 'construction_project_management', 'economics',
+    'education', 'elementary_chinese', 'elementary_commonsense', 'elementary_information_and_technology',
+    'electrical_engineering', 'elementary_mathematics', 'ethnology', 'food_science', 'genetics', 'global_facts',
+    'high_school_biology', 'high_school_chemistry', 'high_school_geography', 'high_school_mathematics',
+    'high_school_physics', 'high_school_politics', 'human_sexuality', 'international_law', 'journalism',
+    'jurisprudence', 'legal_and_moral_basis', 'logical', 'machine_learning', 'management', 'marketing',
+    'marxist_theory', 'modern_chinese', 'nutrition', 'philosophy', 'professional_accounting', 'professional_law',
+    'professional_medicine', 'professional_psychology', 'public_relations', 'security_study', 'sociology',
+    'sports_science', 'traditional_chinese_medicine', 'virology', 'world_history', 'world_religions'
 ]
 
-
-SUBJECT_MAPPING = {"agronomy": ["other", "Other"],
-                   "anatomy": ["biology", "STEM"],
-                   "ancient_chinese": ["china specific", "China specific"],
-                   "arts": ["arts", "Humanities"],
-                   "astronomy": ["physics", "STEM"],
-                   "business_ethics": ["business", "Social Science"],
-                   "chinese_civil_service_exam": ["china specific", "China specific"],
-                   "chinese_driving_rule": ["china specific", "China specific"],
-                   "chinese_food_culture": ["china specific", "China specific"],
-                   "chinese_foreign_policy": ["china specific", "China specific"],
-                   "chinese_history": ["china specific", "China specific"],
-                   "chinese_literature": ["china specific", "China specific"],
-                   "chinese_teacher_qualification": ["china specific", "China specific"],
-                   "college_actuarial_science": ["math", "STEM"],
-                   "college_education": ["education", "Social Science"],
-                   "college_engineering_hydrology": ["engineering", "STEM"],
-                   "college_law": ["law", "Humanities"],
-                   "college_mathematics": ["math", "STEM"],
-                   "college_medical_statistics": ["statistics", "STEM"],
-                   "clinical_knowledge": ["other", "Other"],
-                   "college_medicine": ["other", "Other"],
-                   "computer_science": ["computer science", "STEM"],
-                   "computer_security": ["other", "Other"],
-                   "conceptual_physics": ["physics", "STEM"],
-                   "construction_project_management": ["china specific", "China specific"],
-                   "economics": ["economics", "Social Science"],
-                   "education": ["education", "Social Science"],
-                   "elementary_chinese": ["china specific", "China specific"],
-                   "elementary_commonsense": ["china specific", "China specific"],
-                   "elementary_information_and_technology": ["other", "Other"],
-                   "electrical_engineering": ["engineering", "STEM"],
-                   "elementary_mathematics": ["math", "STEM"],
-                   "ethnology": ["china specific", "China specific"],
-                   "food_science": ["other", "Other"],
-                   "genetics": ["biology", "STEM"],
-                   "global_facts": ["global", "Humanities"],
-                   "high_school_biology": ["biology", "STEM"],
-                   "high_school_chemistry": ["chemistry", "STEM"],
-                   "high_school_geography": ["geography", "Social Science"],
-                   "high_school_mathematics": ["math", "STEM"],
-                   "high_school_physics": ["physics", "STEM"],
-                   "high_school_politics": ["china specific", "China specific"],
-                   "human_sexuality": ["other", "Other"],
-                   "international_law": ["law", "Humanities"],
-                   "journalism": ["sociology", "Social Science"],
-                   "jurisprudence": ["law", "Humanities"],
-                   "legal_and_moral_basis": ["other", "Other"],
-                   "logical": ["philosophy", "Humanities"],
-                   "machine_learning": ["computer science", "STEM"],
-                   "management": ["business", "Social Science"],
-                   "marketing": ["business", "Social Science"],
-                   "marxist_theory": ["philosophy", "Humanities"],
-                   "modern_chinese": ["china specific", "China specific"],
-                   "nutrition": ["other", "Other"],
-                   "philosophy": ["philosophy", "Humanities"],
-                   "professional_accounting": ["business", "Social Science"],
-                   "professional_law": ["law", "Humanities"],
-                   "professional_medicine": ["other", "Other"],
-                   "professional_psychology": ["psychology", "Social Science"],
-                   "public_relations": ["politics", "Social Science"],
-                   "security_study": ["politics", "Social Science"],
-                   "sociology": ["culture", "Social Science"],
-                   "sports_science": ["other", "Other"],
-                   "traditional_chinese_medicine": ["china specific", "China specific"],
-                   "virology": ["biology", "STEM"],
-                   "world_history": ["history", "Humanities"],
-                   "world_religions": ["global", "Humanities"]
+SUBJECT_MAPPING = {
+    'agronomy': ['other', 'Other'],
+    'anatomy': ['biology', 'STEM'],
+    'ancient_chinese': ['china specific', 'China specific'],
+    'arts': ['arts', 'Humanities'],
+    'astronomy': ['physics', 'STEM'],
+    'business_ethics': ['business', 'Social Science'],
+    'chinese_civil_service_exam': ['china specific', 'China specific'],
+    'chinese_driving_rule': ['china specific', 'China specific'],
+    'chinese_food_culture': ['china specific', 'China specific'],
+    'chinese_foreign_policy': ['china specific', 'China specific'],
+    'chinese_history': ['china specific', 'China specific'],
+    'chinese_literature': ['china specific', 'China specific'],
+    'chinese_teacher_qualification': ['china specific', 'China specific'],
+    'college_actuarial_science': ['math', 'STEM'],
+    'college_education': ['education', 'Social Science'],
+    'college_engineering_hydrology': ['engineering', 'STEM'],
+    'college_law': ['law', 'Humanities'],
+    'college_mathematics': ['math', 'STEM'],
+    'college_medical_statistics': ['statistics', 'STEM'],
+    'clinical_knowledge': ['other', 'Other'],
+    'college_medicine': ['other', 'Other'],
+    'computer_science': ['computer science', 'STEM'],
+    'computer_security': ['other', 'Other'],
+    'conceptual_physics': ['physics', 'STEM'],
+    'construction_project_management': ['china specific', 'China specific'],
+    'economics': ['economics', 'Social Science'],
+    'education': ['education', 'Social Science'],
+    'elementary_chinese': ['china specific', 'China specific'],
+    'elementary_commonsense': ['china specific', 'China specific'],
+    'elementary_information_and_technology': ['other', 'Other'],
+    'electrical_engineering': ['engineering', 'STEM'],
+    'elementary_mathematics': ['math', 'STEM'],
+    'ethnology': ['china specific', 'China specific'],
+    'food_science': ['other', 'Other'],
+    'genetics': ['biology', 'STEM'],
+    'global_facts': ['global', 'Humanities'],
+    'high_school_biology': ['biology', 'STEM'],
+    'high_school_chemistry': ['chemistry', 'STEM'],
+    'high_school_geography': ['geography', 'Social Science'],
+    'high_school_mathematics': ['math', 'STEM'],
+    'high_school_physics': ['physics', 'STEM'],
+    'high_school_politics': ['china specific', 'China specific'],
+    'human_sexuality': ['other', 'Other'],
+    'international_law': ['law', 'Humanities'],
+    'journalism': ['sociology', 'Social Science'],
+    'jurisprudence': ['law', 'Humanities'],
+    'legal_and_moral_basis': ['other', 'Other'],
+    'logical': ['philosophy', 'Humanities'],
+    'machine_learning': ['computer science', 'STEM'],
+    'management': ['business', 'Social Science'],
+    'marketing': ['business', 'Social Science'],
+    'marxist_theory': ['philosophy', 'Humanities'],
+    'modern_chinese': ['china specific', 'China specific'],
+    'nutrition': ['other', 'Other'],
+    'philosophy': ['philosophy', 'Humanities'],
+    'professional_accounting': ['business', 'Social Science'],
+    'professional_law': ['law', 'Humanities'],
+    'professional_medicine': ['other', 'Other'],
+    'professional_psychology': ['psychology', 'Social Science'],
+    'public_relations': ['politics', 'Social Science'],
+    'security_study': ['politics', 'Social Science'],
+    'sociology': ['culture', 'Social Science'],
+    'sports_science': ['other', 'Other'],
+    'traditional_chinese_medicine': ['china specific', 'China specific'],
+    'virology': ['biology', 'STEM'],
+    'world_history': ['history', 'Humanities'],
+    'world_religions': ['global', 'Humanities']
 }
 
 
@@ -171,12 +119,13 @@ class CMMLUAdapter(DataAdapter):
         if metric_list is None:
             metric_list = [{'name': 'WeightedAverageAccuracy', 'object': weighted_mean}]
 
-        super().__init__(subset_list=subset_list,
-                        metric_list=metric_list,
-                        few_shot_num=few_shot_num,
-                        train_split=train_split,
-                        eval_split=eval_split,
-                        **kwargs)
+        super().__init__(
+            subset_list=subset_list,
+            metric_list=metric_list,
+            few_shot_num=few_shot_num,
+            train_split=train_split,
+            eval_split=eval_split,
+            **kwargs)
 
     def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
         data_dict = {}
@@ -223,9 +172,7 @@ class CMMLUAdapter(DataAdapter):
             {'data': [(context, continuation), ...]}
 
         """
-        prompt = '以下是关于{}的单项选择题。\n\n'.format(
-            self._format_subject(subset_name)
-        )
+        prompt = '以下是关于{}的单项选择题。\n\n'.format(self._format_subject(subset_name))
         few_shot_prompts = [self._generate_prompt(input_d=sample, include_answer=True) for sample in few_shot_list]
 
         context: str = '\n'.join(few_shot_prompts) + '\n'
@@ -331,17 +278,24 @@ class CMMLUAdapter(DataAdapter):
             domain_weighted_avg_acc = sum([score * num for _, score, num in domain_res_list]) / \
                                      sum([num for _, _, num in domain_res_list])
             domain_weighted_avg_acc = normalize_score(score=domain_weighted_avg_acc)
-            category_list.append({'name': domain_name,
-                                  'score': domain_weighted_avg_acc,
-                                  'subset': [{'name': subset_name, 'score': normalize_score(subset_score)}
-                                             for subset_name, subset_score, _ in domain_res_list]})
+            category_list.append({
+                'name':
+                domain_name,
+                'score':
+                domain_weighted_avg_acc,
+                'subset': [{
+                    'name': subset_name,
+                    'score': normalize_score(subset_score)
+                } for subset_name, subset_score, _ in domain_res_list]
+            })
 
         # Get final dict of report
-        res_map = dict(name=report_name or 'cmmlu',
-                       metric=self.metric_list[0]['name'],
-                       score=weighted_avg_acc,
-                       category=category_list,
-                       total_num=total_num)
+        res_map = dict(
+            name=report_name or 'cmmlu',
+            metric=self.metric_list[0]['name'],
+            score=weighted_avg_acc,
+            category=category_list,
+            total_num=total_num)
 
         return res_map
 
@@ -366,4 +320,4 @@ class CMMLUAdapter(DataAdapter):
         s = ''
         for entry in l:
             s += ' ' + entry
-        return s
+        return s

evalscope/benchmarks/cmmlu/samples.jsonl

@@ -2,4 +2,4 @@
 {'input': '下列关于重力的说法正确的是', 'A': '在地球周围的物体都要受到重力作用，与其运动状态无关', 'B': '对某一物体而言，重力的大小是一个恒量，不随物体的地理位置而改变', 'C': '重力就是地球对物体的吸引力，重力的方向总是竖直向下', 'D': '在地球表面各处的重力方向都是相同的', 'target': 'A'}
 {'input': '心脏的静脉血回心的主要途径是', 'A': '心小静脉', 'B': '冠状窦', 'C': '心中静脉', 'D': '心前静脉', 'target': 'B'}
 {'input': "以西蒙为代表的决策理论学派提出的决策准则是", 'A': '最优化', 'B': '公平', 'C': '民主化', 'D': '满意', 'target': 'D'}
-{'input': '20世纪初，英国首相阿斯奎斯说：“我们现在有一个牢固确立了两百年的传统，即归根到底，王位的占有者接受其大臣的建议并据此行事。”这一传统的确立，使一个以小农业和手工业生产为主的国家变成了一个典型的资本主义国家，成为欧洲各国效仿的对象。各国效仿的理由是', 'A': '英国“光荣革命”宣告了欧洲新社会政治制度的诞生', 'B': '殖民主义深刻影响了英国“世界工厂”的地位', 'C': '英国经济上的成就得益于其制度设计', 'D': '英国启蒙思想奠定了资产阶级民主主义政治的理论基础', 'target': 'C'}
+{'input': '20世纪初，英国首相阿斯奎斯说：“我们现在有一个牢固确立了两百年的传统，即归根到底，王位的占有者接受其大臣的建议并据此行事。”这一传统的确立，使一个以小农业和手工业生产为主的国家变成了一个典型的资本主义国家，成为欧洲各国效仿的对象。各国效仿的理由是', 'A': '英国“光荣革命”宣告了欧洲新社会政治制度的诞生', 'B': '殖民主义深刻影响了英国“世界工厂”的地位', 'C': '英国经济上的成就得益于其制度设计', 'D': '英国启蒙思想奠定了资产阶级民主主义政治的理论基础', 'target': 'C'}

evalscope/benchmarks/competition_math/__init__.py

@@ -1,5 +1,6 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-from evalscope.benchmarks.competition_math.competition_math_adapter import CompetitionMathAdapter, DATASET_ID, SUBSET_LIST
+from evalscope.benchmarks.competition_math.competition_math_adapter import DATASET_ID, SUBSET_LIST
+from evalscope.benchmarks.competition_math.competition_math_adapter import CompetitionMathAdapter
 from evalscope.benchmarks.competition_math.competition_math_adapter import CompetitionMathAdapter as DataAdapterClass
-from evalscope.models.model_adapter import ChatGenerationModelAdapter as ModelAdapterClass    # noqa
+from evalscope.models.model_adapter import ChatGenerationModelAdapter as ModelAdapterClass  # noqa

evalscope/benchmarks/competition_math/competition_math.py

@@ -1,13 +1,10 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
-
 """Mathematics Aptitude Test of Heuristics (MATH) dataset."""
 
+import datasets
 import json
 import os
 
-import datasets
-
-
 _CITATION = """\
 @article{hendrycksmath2021,
   title={Measuring Mathematical Problem Solving With the MATH Dataset},
@@ -24,7 +21,6 @@ _CITATION = """\
 }
 """
 
-
 _DESCRIPTION = """\
 The Mathematics Aptitude Test of Heuristics (MATH) dataset consists of problems
 from mathematics competitions, including the AMC 10, AMC 12, AIME, and more.
@@ -32,13 +28,10 @@ Each problem in MATH has a full step-by-step solution, which can be used to teac
 models to generate answer derivations and explanations.
 """
 
-
 _HOMEPAGE = 'https://github.com/hendrycks/math'
 
-
 _LICENSE = 'https://github.com/hendrycks/math/blob/main/LICENSE'
 
-
 # Original data URL: "https://people.eecs.berkeley.edu/~hendrycks/MATH.tar"
 _URL = 'https://sail-moe.oss-cn-hangzhou.aliyuncs.com/open_data/math/MATH.zip'
 
@@ -49,14 +42,12 @@ class CompetitionMathDataset(datasets.GeneratorBasedBuilder):
     VERSION = datasets.Version('1.0.0')
 
     def _info(self):
-        features = datasets.Features(
-            {
-                'problem': datasets.Value('string'),
-                'level': datasets.Value('string'),
-                'type': datasets.Value('string'),
-                'solution': datasets.Value('string'),
-            }
-        )
+        features = datasets.Features({
+            'problem': datasets.Value('string'),
+            'level': datasets.Value('string'),
+            'type': datasets.Value('string'),
+            'solution': datasets.Value('string'),
+        })
         return datasets.DatasetInfo(
             description=_DESCRIPTION,
             features=features,

evalscope/benchmarks/competition_math/competition_math_adapter.py

@@ -8,6 +8,7 @@ from evalscope.benchmarks import DataAdapter
 from evalscope.metrics.metrics import weighted_mean
 from evalscope.utils import normalize_score
 from evalscope.utils.logger import get_logger
+
 # flake8: noqa
 
 logger = get_logger()
@@ -43,12 +44,13 @@ class CompetitionMathAdapter(DataAdapter):
                          f'but got {self.few_shot_num}. Use 4-shot by default.')
             few_shot_num = 4
 
-        super().__init__(subset_list=subset_list,
-                         metric_list=metric_list,
-                         few_shot_num=few_shot_num,
-                         train_split=train_split,
-                         eval_split=eval_split,
-                         **kwargs)
+        super().__init__(
+            subset_list=subset_list,
+            metric_list=metric_list,
+            few_shot_num=few_shot_num,
+            train_split=train_split,
+            eval_split=eval_split,
+            **kwargs)
 
     def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
         data_dict: dict = {}
@@ -161,17 +163,19 @@ class CompetitionMathAdapter(DataAdapter):
         total_num: int = sum([num for _, num in subset_score_map.values()])
         weighted_avg_acc: float = sum([score * num for score, num in subset_score_map.values()]) / total_num
         weighted_avg_acc = normalize_score(score=weighted_avg_acc)
-        cate_avg_list = [{'name': subset_name, 'score': normalize_score(score=score)} for subset_name, (score, _) in subset_score_map.items()]
+        cate_avg_list = [{
+            'name': subset_name,
+            'score': normalize_score(score=score)
+        } for subset_name, (score, _) in subset_score_map.items()]
 
-        category_d = dict(name='DEFAULT',
-                          score=weighted_avg_acc,
-                          subset=cate_avg_list)
+        category_d = dict(name='DEFAULT', score=weighted_avg_acc, subset=cate_avg_list)
 
-        res_map = dict(name=report_name or 'competition_math',
-                       metric=self.metric_list[0]['name'],
-                       score=weighted_avg_acc,
-                       category=[category_d],
-                       total_num=total_num)
+        res_map = dict(
+            name=report_name or 'competition_math',
+            metric=self.metric_list[0]['name'],
+            score=weighted_avg_acc,
+            category=[category_d],
+            total_num=total_num)
 
         return res_map
 
@@ -186,8 +190,7 @@ class CompetitionMathAdapter(DataAdapter):
                 'Problem:\nIf $\det \mathbf{{A}} = 2$ and $\det \mathbf{{B}} = 12,$ then find $\det (\mathbf{{A}} \mathbf{{B}}).$\nSolution:\nWe have that $\det (\mathbf{{A}} \mathbf{{B}}) = (\det \mathbf{{A}})(\det \mathbf{{B}}) = (2)(12) = \\boxed{{24}}.$\nFinal Answer: The final answer is $24$. I hope it is correct.\n'
                 'Problem:\nTerrell usually lifts two 20-pound weights 12 times. If he uses two 15-pound weights instead, how many times must Terrell lift them in order to lift the same total weight?\nSolution:\nIf Terrell lifts two 20-pound weights 12 times, he lifts a total of $2\cdot 12\cdot20=480$ pounds of weight. If he lifts two 15-pound weights instead for $n$ times, he will lift a total of $2\cdot15\cdot n=30n$ pounds of weight. Equating this to 480 pounds, we can solve for $n$: \\begin{{align*}} 30n&=480\\\\ \Rightarrow\qquad n&=480/30=\\boxed{{16}} \end{{align*}}\nFinal Answer: The final answer is $16$. I hope it is correct.\n'
                 'Problem:\nIf the system of equations: \\begin{{align*}} 6x-4y&=a,\\\\ 6y-9x &=b. \end{{align*}}has a solution $(x, y)$ where $x$ and $y$ are both nonzero, find $\\frac{{a}}{{b}},$ assuming $b$ is nonzero.\nSolution:\nIf we multiply the first equation by $-\\frac{{3}}{{2}}$, we obtain $$6y-9x=-\\frac{{3}}{{2}}a.$$Since we also know that $6y-9x=b$, we have $$-\\frac{{3}}{{2}}a=b\Rightarrow\\frac{{a}}{{b}}=\\boxed{{-\\frac{{2}}{{3}}}}.$$\nFinal Answer: The final answer is $-\\frac{{2}}{{3}}$. I hope it is correct.\n'
-                f'Problem:\n{problem}\nSolution:\n'
-            )
+                f'Problem:\n{problem}\nSolution:\n')
         else:
             context = 'Problem:\n' + problem + '\nSolution:\n'
         return context
@@ -212,15 +215,15 @@ class CompetitionMathAdapter(DataAdapter):
 
         if '\\boxed ' in s:
             left = '\\boxed '
-            assert s[: len(left)] == left
+            assert s[:len(left)] == left
             return s[len(left):]
 
         left = '\\boxed{'
 
-        assert s[: len(left)] == left
+        assert s[:len(left)] == left
         assert s[-1] == '}'
 
-        return s[len(left): -1]
+        return s[len(left):-1]
 
     @classmethod
     def _last_boxed_only_string(cls, string):
@@ -249,7 +252,7 @@ class CompetitionMathAdapter(DataAdapter):
         if right_brace_idx is None:
             retval = None
         else:
-            retval = string[idx: right_brace_idx + 1]
+            retval = string[idx:right_brace_idx + 1]
 
         return retval
 
@@ -409,18 +412,14 @@ class CompetitionMathAdapter(DataAdapter):
 
     @classmethod
     def _math_postprocess(cls, text: str) -> str:
-        SUBSTITUTIONS = [('an ', ''), ('a ', ''), ('.$', '$'), ('\\$', ''),
-                         (r'\ ', ''), (' ', ''), ('mbox', 'text'),
-                         (',\\text{and}', ','), ('\\text{and}', ','),
-                         ('\\text{m}', '\\text{}'), ('\\le', '<')]
+        SUBSTITUTIONS = [('an ', ''), ('a ', ''), ('.$', '$'), ('\\$', ''), (r'\ ', ''), (' ', ''), ('mbox', 'text'),
+                         (',\\text{and}', ','), ('\\text{and}', ','), ('\\text{m}', '\\text{}'), ('\\le', '<')]
         REMOVED_EXPRESSIONS = [
-            'square', 'ways', 'integers', 'dollars', 'mph', 'inches', 'ft',
-            'hours', 'km', 'units', '\\ldots', 'sue', 'points', 'feet', 'minutes',
-            'digits', 'cents', 'degrees', 'cm', 'gm', 'pounds', 'meters', 'meals',
-            'edges', 'students', 'childrentickets', 'multiples', '\\text{s}',
-            '\\text{.}', '\\text{\ns}', '\\text{}^2', '\\text{}^3', '\\text{\n}',
-            '\\text{}', r'\mathrm{th}', r'^\circ', r'^{\circ}', r'\;', r',\!',
-            '{,}', '"', '\\dots', '\n', '\r', '\f'
+            'square', 'ways', 'integers', 'dollars', 'mph', 'inches', 'ft', 'hours', 'km', 'units', '\\ldots', 'sue',
+            'points', 'feet', 'minutes', 'digits', 'cents', 'degrees', 'cm', 'gm', 'pounds', 'meters', 'meals', 'edges',
+            'students', 'childrentickets', 'multiples', '\\text{s}', '\\text{.}', '\\text{\ns}', '\\text{}^2',
+            '\\text{}^3', '\\text{\n}', '\\text{}', r'\mathrm{th}', r'^\circ', r'^{\circ}', r'\;', r',\!', '{,}', '"',
+            '\\dots', '\n', '\r', '\f'
         ]
         import re
 
@@ -453,8 +452,7 @@ class CompetitionMathAdapter(DataAdapter):
             if 'rac' in final_answer and '\\frac' not in final_answer:
                 final_answer = final_answer.replace('rac', '\\frac')
 
-            final_answer = re.sub(r'(frac)([^{])(.)', 'frac{\\2}{\\3}',
-                                  final_answer)
+            final_answer = re.sub(r'(frac)([^{])(.)', 'frac{\\2}{\\3}', final_answer)
             final_answer = re.sub(r'(sqrt)([^{])', 'sqrt{\\2}', final_answer)
             final_answer = final_answer.replace('$', '')