tasksource 0.0.43__tar.gz → 0.0.46__tar.gz

{tasksource-0.0.43 → tasksource-0.0.46}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tasksource
-Version: 0.0.43
+Version: 0.0.46
 Summary: Preprocessings to prepare datasets for a task
 Home-page: https://github.com/sileod/tasksource/
 Classifier: Programming Language :: Python :: 3

{tasksource-0.0.43 → tasksource-0.0.46}/src/tasksource/.ipynb_checkpoints/preprocess-checkpoint.py

@@ -11,6 +11,8 @@ import copy
 import datasets
 import time
 
+MAX_MC_OPTIONS = 4
+
 def get_column_names(dataset):
     cn = dataset.column_names
     if type(cn)==dict:
@@ -28,6 +30,11 @@ def sample_dataset(dataset,n=10000, n_eval=1000,seed=0):
 
 class Preprocessing(DotWiz):
     default_splits = ('train','validation','test')
+    _instances = []
+
+    def __post_init__(self):
+        Preprocessing._instances+=[self]
+
     @staticmethod
     def __map_to_target(x,fn=lambda x:None, target=None):
         x[target]=fn(x)
@@ -158,12 +165,15 @@ class MultipleChoiceFields(Preprocessing):
         if self.choices_list:
             dataset = dataset.filter(lambda x: 1<len(x['choices_list']))
             n_options = min([len(x) for k in dataset for x in dataset[k]['choices_list']])
-            n_options = min(5,n_options)
-            dataset = dataset.map(self.flatten, fn_kwargs={'n_options':n_options})
+            n_options = min(MAX_MC_OPTIONS,n_options)
+            dataset = dataset.map(self.flatten_choice_list, fn_kwargs={'n_options':n_options})
+
+        else:
+            dataset = dataset.map(self.sample_choices, fn_kwargs={'n_options':MAX_MC_OPTIONS})
         return dataset
 
     @staticmethod
-    def flatten(x, n_options=None):
+    def flatten_choice_list(x, n_options=None):
         n_neg = n_options-1 if n_options else None
         choices = x['choices_list']
         label=x['labels']
@@ -176,13 +186,31 @@ class MultipleChoiceFields(Preprocessing):
         del x['choices_list']
         return x
 
+    @staticmethod
+    def sample_choices(x, n_options=None):
+        choices = [x[c] for c in x if 'choice' in c]
+        if not MAX_MC_OPTIONS or len(choices)<=n_options:
+            return x
+        n_neg = n_options-1 if n_options else None
+        label=x['labels']
+        neg = choices[:label] + choices[label+1:]
+        pos = choices[label]
+        x['labels']=0
+        choices_list=[pos]+neg[:n_neg]
+        for c in list(x):
+            if 'choice' in c:
+                del x[c]
+        for i,o in enumerate(choices_list):
+            x[f'choice{i}']=o
+        return x
+
 @dataclass
 class SharedFields:
     splits:list=Preprocessing.default_splits
     dataset_name:str = None
     config_name:str = None
-    pre_process: callable = lambda x:x
-    post_process: callable = lambda x:x
+    pre_process: callable = fc.identity
+    post_process: callable = fc.identity
     #language:str="en"
     
 

{tasksource-0.0.43 → tasksource-0.0.46}/src/tasksource/.ipynb_checkpoints/recast-checkpoint.py

@@ -8,12 +8,14 @@ improper_labels += ['glue/stsb', 'sick/relatedness', 'joci', 'utilitarianism', '
 
 improper_labels += ['stsb_multi_mt','MLMA_hate_speech','icl-symbol-tuning-instruct','zero-shot-label-nli']
 
+improper_labels += ['essay-scoring','english-grading','HelpSteer','oasst2']
+
 def render_options(options):
     options = [f'"{x}"' for x in options]
     return f"{', '.join(options[:-1])} or {options[-1]}"
 
 def render_classification(text,options,answer):
-    example = 'A→B' if text.startswith('A:') else 'the following'
+    example = 'text_A→text_B' if text.startswith('text_A:') else 'the following'
     inputs = f'With no explanation, label {example} with either {render_options(options)}.\n{text}'
     targets = f"{answer}."
     return dict_of(inputs,targets)
@@ -99,7 +101,7 @@ def recast_instruct(dataset):
 
     def recast_Classification(x):
         if 'sentence2' in x:
-            text=f"A: {x['sentence1']}\nB: {x['sentence2']}"
+            text=f"text_A: {x['sentence1']}\ntext_B: {x['sentence2']}"
         else:
             text=x['sentence1']
             

{tasksource-0.0.43 → tasksource-0.0.46}/src/tasksource/.ipynb_checkpoints/tasks-checkpoint.py

@@ -38,8 +38,8 @@ anli__a3 = Classification('premise','hypothesis','label', splits=['train_r3','de
 
 
 babi_nli = Classification("premise", "hypothesis", "label",
-    dataset_name="metaeval/babi_nli",
-    config_name=set(get_dataset_config_names("metaeval/babi_nli"))-{"agents-motivations"}
+    dataset_name="tasksource/babi_nli",
+    config_name=set(get_dataset_config_names("tasksource/babi_nli"))-{"agents-motivations"}
 ) # agents-motivations task is not as clear-cut as the others
 
 
@@ -60,7 +60,7 @@ hans = Classification(sentence1="premise", sentence2="hypothesis", labels="label
 
 wanli = Classification('premise','hypothesis','gold', dataset_name="alisawuffles/WANLI")
 
-recast_nli = Classification(sentence1="context", sentence2="hypothesis", labels="label", dataset_name="metaeval/recast",
+recast_nli = Classification(sentence1="context", sentence2="hypothesis", labels="label", dataset_name="tasksource/recast",
     config_name=['recast_kg_relations', 'recast_puns', 'recast_factuality', 'recast_verbnet',
     'recast_verbcorner', 'recast_ner', 'recast_sentiment', 'recast_megaveridicality'])
 
@@ -69,7 +69,7 @@ probability_words_nli = Classification(sentence1="context", sentence2="hypothesi
     dataset_name="sileod/probability_words_nli", 
     config_name=["reasoning_1hop","reasoning_2hop","usnli"])
 
-nan_nli = Classification("premise", "hypothesis", "label", dataset_name="joey234/nan-nli", config_name="joey234--nan-nli")
+nan_nli = Classification("premise", "hypothesis", "label", dataset_name="joey234/nan-nli")
 
 nli_fever = Classification("premise","hypothesis","label",
     dataset_name="pietrolesci/nli_fever", splits=["train","dev",None])
@@ -153,15 +153,15 @@ def _imppres_post_process(ds,prefix=''):
     names=[f'{prefix}_entailment',f'{prefix}_neutral',f'{prefix}_contradiction']))
 
 imppres__presupposition = imppres__prag = Classification("premise","hypothesis","gold_label",
-    dataset_name="metaeval/imppres", config_name=imppres_presupposition,
+    dataset_name="tasksource/imppres", config_name=imppres_presupposition,
     post_process=_imppres_post_process)
 
 imppres__prag = Classification("premise","hypothesis","gold_label_prag",
-    dataset_name="metaeval/imppres", config_name=imppres_implicature,
+    dataset_name="tasksource/imppres", config_name=imppres_implicature,
     post_process=lambda x: _imppres_post_process(x,'pragmatic'))
 
 imppres__log = Classification("premise","hypothesis","gold_label_log",
-    dataset_name="metaeval/imppres", config_name=imppres_implicature,
+    dataset_name="tasksource/imppres", config_name=imppres_implicature,
     post_process=lambda x: _imppres_post_process(x,'logical'))
 
 
@@ -212,6 +212,7 @@ bigbench = MultipleChoice(
     dataset_name='tasksource/bigbench',
     config_name=bigbench_discriminative_english - {"social_i_qa","intersect_geometry"} # english multiple choice tasks, minus duplicates
 )
+#"goal_step_wikihow"
 
 blimp_hard = MultipleChoice(inputs=constant(''),
     choices=['sentence_good','sentence_bad'],
@@ -367,7 +368,7 @@ turingbench = Classification("Generation",labels="label",
 
 trec = Classification(sentence1="text", labels="fine_label")
 
-tals_vitaminc = Classification('claim','evidence','label', dataset_name="tals/vitaminc", config_name="tals--vitaminc")
+tals_vitaminc = Classification('claim','evidence','label', dataset_name="tals/vitaminc")
 
 hope_edi = Classification("text", labels="label", splits=["train", "validation", None], config_name=["english"])
 
@@ -426,8 +427,6 @@ silicone = Classification("Utterance",labels="Label",
     config_name=['dyda_da', 'dyda_e', 'iemocap', 'maptask', 'meld_e', 'meld_s', 'oasis', 'sem'] # +['swda', 'mrda'] # in pragmeval
 )
 
-#lex_glue___ecthr_a = Classification(sentence1="text", labels="labels") # too long
-#lex_glue___ecthr_b = Classification(sentence1="text", labels="labels") # too long
 lex_glue___eurlex = Classification(sentence1="text", labels="labels") 
 lex_glue___scotus = Classification(sentence1="text", labels="label")
 lex_glue___ledgar = Classification(sentence1="text", labels="label")
@@ -485,7 +484,11 @@ stackoverflow_questions=Classification("title","body",labels="label",
 
 #hyperpartisan_news_detection___byarticle = Classification(sentence1="text", labels="hyperpartisan", splits=["train", None, None]) # files too heavy
 #hyperpartisan_news_detection___bypublisher = Classification(sentence1="text", labels="hyperpartisan", splits=["train","validation", None]) # files too heavy
-hyperpartisan_news = Classification("text",labels="label",dataset_name="zapsdcn/hyperpartisan_news")
+
+hyperpartisan_news = Classification(
+    "text",
+    labels=lambda x: {'true':'hyperpartisan','false':'not_hyperpartisan'}.get(x["label"]),
+    dataset_name="zapsdcn/hyperpartisan_news")
 
 scierc = Classification("text",labels="label",dataset_name="zapsdcn/sciie")
 citation_intent = Classification("text",labels="label",dataset_name="zapsdcn/citation_intent")
@@ -508,7 +511,7 @@ relbert_lexical_relation_classification = Classification(sentence1="head", sente
  config_name=["BLESS","CogALexV","EVALution","K&H+N","ROOT09"])
 
 
-metaeval_linguisticprobing = Classification("sentence", labels="label", dataset_name="metaeval/linguisticprobing", 
+linguisticprobing = Classification("sentence", labels="label", dataset_name="tasksource/linguisticprobing", 
     config_name=['subj_number',
                 'obj_number',
                 'past_present',
@@ -635,7 +638,7 @@ quarel = Classification(
     labels=lambda x: "AB"[x["answer_index"]]
 )
 
-mwong_fever_evidence_related = Classification(sentence1="claim", sentence2="evidence", labels="labels", splits=["train", "valid", "test"], dataset_name="mwong/fever-evidence-related", config_name="mwong--fever-related")
+mwong_fever_evidence_related = Classification(sentence1="claim", sentence2="evidence", labels="labels", splits=["train", "valid", "test"], dataset_name="mwong/fever-evidence-related")
 
 numer_sense = Classification("sentence",labels="target",splits=["train",None,None])
 
@@ -649,7 +652,9 @@ sarcasm_news = Classification("headline", labels="is_sarcastic",
 
 sem_eval_2010_task_8 = Classification("sentence",labels="relation")
 
-demo_org_auditor_review = Classification(sentence1="sentence", labels="label", splits=["train", None, "test"], dataset_name="demo-org/auditor_review", config_name="demo-org--auditor_review")
+auditor_review = Classification(sentence1="sentence",
+    labels=name("label",['negative','neutral','positive']),
+    dataset_name="demo-org/auditor_review")
 
 medmcqa = MultipleChoice("question", choices=regen('op[a-d]'),labels='cop')
 
@@ -677,9 +682,9 @@ logiqa = MultipleChoice(
 wiki_qa = Classification("question","answer", name("label",['False','True']))
 
 cycic_classification = Classification("question",labels=name("correct_answer",['False','True']),
-    dataset_name = "metaeval/cycic_classification")
+    dataset_name = "tasksource/cycic_classification")
 cycic_mc = MultipleChoice("question", choices=regen('answer\_option[0-4]'), labels="correct_answer",
-    dataset_name = "metaeval/cycic_multiplechoice")
+    dataset_name = "tasksource/cycic_multiplechoice")
 
 
 def _preprocess_chatgpt_detection(ex):
@@ -694,24 +699,24 @@ def _preprocess_chatgpt_detection(ex):
 #    pre_process=lambda dataset:dataset.map(_preprocess_chatgpt_detection))
 
 sts_companion = Classification("sentence1","sentence2","label",
-    dataset_name="metaeval/sts-companion")
+    dataset_name="tasksource/sts-companion")
 
 commonsense_qa_2 = Classification("question",labels="answer",
-    dataset_name="metaeval/commonsense_qa_2.0")
+    dataset_name="tasksource/commonsense_qa_2.0")
 
-ling_nli = Classification("premise","hypothesis","label",dataset_name="metaeval/lingnli")
+ling_nli = Classification("premise","hypothesis","label",dataset_name="tasksource/lingnli")
 
 monotonicity_entailment = Classification("sentence1", "sentence2", "gold_label",    
-    dataset_name="metaeval/monotonicity-entailment")
+    dataset_name="tasksource/monotonicity-entailment")
 
 arct = MultipleChoice(cat(["reason","claim"]),choices=["warrant0","warrant1"],
-    labels="correctLabelW0orW1", dataset_name="metaeval/arct")
+    labels="correctLabelW0orW1", dataset_name="tasksource/arct")
 
 scinli = Classification("sentence1", "sentence2", labels="label",
     post_process=lambda x:x.shuffle(seed=0),
-    dataset_name="metaeval/scinli")
+    dataset_name="tasksource/scinli")
 
-naturallogic = Classification(" sent1 "," sent2 "," new_label ",dataset_name="metaeval/naturallogic")
+naturallogic = Classification(" sent1 "," sent2 "," new_label ",dataset_name="tasksource/naturallogic")
 
 onestop_qa = MultipleChoice(cat(["paragraph","question"]),choices_list="answers",
     labels=constant(0))
@@ -727,7 +732,7 @@ dyna_hate = Classification("text",labels="label",dataset_name="aps/dynahate",spl
 
 syntactic_augmentation_nli = Classification('sentence1',"sentence2","gold_label",dataset_name="metaeval/syntactic-augmentation-nli")
 
-autotnli = Classification("premises", "hypothesis", "label", dataset_name="metaeval/autotnli")
+autotnli = Classification("premises", "hypothesis", "label", dataset_name="tasksource/autotnli")
 #equate = Classification("sentence1", "sentence2", "gold_label",dataset_name="metaeval/equate")
 
 conqada = Classification("sentence1","sentence2","label",dataset_name="lasha-nlp/CONDAQA",
@@ -747,9 +752,9 @@ wouldyourather = MultipleChoice(constant('Most people would rather:'), choices=[
     labels= lambda x: int(x['votes_a']<x['votes_b']),
     dataset_name="metaeval/wouldyourather")
 
-attempto_nli = Classification("premise","hypothesis",
-    lambda x:f'race-{x["race_label"]}',
-    dataset_name="sileod/attempto-nli")
+#attempto_nli = Classification("premise","hypothesis",
+#    lambda x:f'race-{x["race_label"]}',
+#    dataset_name="sileod/attempto-nli")
 
 defeasible_nli = Classification(cat(["Premise","Hypothesis"]),"Update",labels="UpdateType",
     dataset_name="metaeval/defeasible-nli",config_name=['atomic', 'snli'])
@@ -758,25 +763,22 @@ defeasible_nli = Classification(cat(["Premise","Hypothesis"]),"Update",labels="U
 #    dataset_name="metaeval/defeasible-nli",config_name='social')
 
 help_nli = Classification("ori_sentence","new_sentence","gold_label",
-    dataset_name="metaeval/help-nli")
+    dataset_name="tasksource/help-nli")
     
 nli_veridicality_transitivity = Classification("sentence1","sentence2","gold_label",
     dataset_name="metaeval/nli-veridicality-transitivity")
 
-nl_satisfiability= Classification("sentence",labels="label",
-    dataset_name="metaeval/natural-language-satisfiability")
-
 lonli = Classification("premise","hypothesis","label",
-    dataset_name="metaeval/lonli")
+    dataset_name="tasksource/lonli")
 
 dadc_limit = Classification("sentence1","sentence2","label",
-    dataset_name="metaeval/dadc-limit-nli")
+    dataset_name="tasksource/dadc-limit-nli")
 
 flute = Classification("premise","hypothesis","label",
     dataset_name="ColumbiaNLP/FLUTE")
 
 strategy_qa = Classification('question',labels='answer',
-    dataset_name="metaeval/strategy-qa",splits=['train',None,None])
+    dataset_name="tasksource/strategy-qa",splits=['train',None,None])
 
 summarize_from_feedback = MultipleChoice(get.info.post,
     choices_list=lambda x: [x['summaries'][0]['text'],x['summaries'][1]['text']],
@@ -790,10 +792,10 @@ folio = Classification("premises","conclusion",
     dataset_name="tasksource/folio")
 
 tomi_nli = Classification("premise","hypothesis","label",
-    dataset_name="metaeval/tomi-nli")
+    dataset_name="tasksource/tomi-nli")
 
 avicenna = Classification("Premise 1","Premise 2","Syllogistic relation",
-    dataset_name="metaeval/avicenna")
+    dataset_name="tasksource/avicenna")
 
 shp = MultipleChoice("history",
     choices=['human_ref_A','human_ref_B'],
@@ -816,7 +818,7 @@ creak = Classification("sentence",labels="label",
 
 mutual = MultipleChoice("article",choices_list="options",
     labels=lambda x: "ABCD".index(x['answers']),
-    dataset_name="metaeval/mutual",splits=["train",None,None])
+    dataset_name="tasksource/mutual",splits=["train",None,None])
 
 neqa = MultipleChoice('prompt',choices_list='classes',labels="answer_index",
     dataset_name="inverse-scaling/NeQA")
@@ -826,35 +828,35 @@ redefine_math = MultipleChoice('prompt',choices_list='classes',labels="answer_in
     dataset_name="inverse-scaling/redefine-math")
 
 puzzte = Classification("puzzle_text","question","answer",
-    dataset_name="metaeval/puzzte")
+    dataset_name="tasksource/puzzte")
 
 implicatures = MultipleChoice(cat(['context','response'],"\n"),
     choices=['correct_implicature','incorrect_implicature'],
     labels=constant(0),
-    dataset_name='metaeval/implicatures')
+    dataset_name='tasksource/implicatures')
 
 race = MultipleChoice(cat(['question','article'],'\n'), choices_list='options',
     labels=lambda x:'ABCDE'.index(x['answer']),
     config_name=['middle','high'])
 
 race_c = MultipleChoice(cat(['question','article'],'\n'),choices_list='option',labels='label',
-    dataset_name='metaeval/race-c')
+    dataset_name='tasksource/race-c')
 
 spartqa_yn=Classification("story","question","answer",
-    dataset_name="metaeval/spartqa-yn")
+    dataset_name="tasksource/spartqa-yn")
 
 spartqa_mc=MultipleChoice(cat(["story","question"]),choices_list="candidate_answers",labels="answer",
-    dataset_name="metaeval/spartqa-mchoice")
+    dataset_name="tasksource/spartqa-mchoice")
 
 temporal_nli = Classification("Premise","Hypothesis","Label",
-    dataset_name="metaeval/temporal-nli")
+    dataset_name="tasksource/temporal-nli")
 
 riddle_sense = MultipleChoice("question", choices_list=get.choices.text, 
     labels=lambda x : "ABCDE".index(x['answerKey']))
 
 clcd = Classification(
     "sentence1","sentence2","label",
-    dataset_name="metaeval/clcd-english")
+    dataset_name="tasksource/clcd-english")
 
 twentyquestions = Classification("question","subject","answer",dataset_name="maximedb/twentyquestions")
 
@@ -862,16 +864,16 @@ reclor = MultipleChoice(cat(["context","question"]),choices_list="answers",label
     dataset_name="metaeval/reclor",splits=['train','validation',None])
 
 c_aug_imdb = Classification("Text",labels="Sentiment",
-    dataset_name='metaeval/counterfactually-augmented-imdb')
+    dataset_name='tasksource/counterfactually-augmented-imdb')
 
 c_aug_snli = Classification("sentence1","sentence2","gold_label",
-    dataset_name='metaeval/counterfactually-augmented-snli')
+    dataset_name='tasksource/counterfactually-augmented-snli')
 
 cnli = Classification("premise","hypothesis","label",
     dataset_name='metaeval/cnli')
 
 perturbed_boolq = Classification("question",labels="hard_label",
-    dataset_name='metaeval/boolq-natural-perturbations')
+    dataset_name='tasksource/boolq-natural-perturbations')
 
 #mega_acceptability = Classification("sentence",labels="average",
 #    dataset_name='metaeval/mega-acceptability-v2')
@@ -883,14 +885,14 @@ equate = Classification("sentence1","sentence2","gold_label",
     dataset_name='metaeval/equate')
 
 science_qa = MultipleChoice("question",choices_list="choices",labels="answer",
-    dataset_name="metaeval/ScienceQA_text_only")
+    dataset_name="tasksource/ScienceQA_text_only")
 
 ekar=MultipleChoice("question",choices_list=get.choices.text,
     labels=lambda x:"ABCD".index(x['answerKey']),
 dataset_name="Jiangjie/ekar_english")
 
 implicit_hate = Classification("post",labels="class",
-    dataset_name="metaeval/implicit-hate-stg1")
+    dataset_name="tasksource/implicit-hate-stg1")
 
 nli_unambiguity = Classification("premise","hypothesis","gini",
     dataset_name="metaeval/chaos-mnli-ambiguity")
@@ -898,7 +900,7 @@ nli_unambiguity = Classification("premise","hypothesis","gini",
 headline_cause = Classification('left_title','right_title','label',
     dataset_name='IlyaGusev/headline_cause',config_name='en_simple')
 
-logiqa_2 = Classification("premise","hypothesis","label",dataset_name="metaeval/logiqa-2.0-nli")
+logiqa_2 = Classification("premise","hypothesis","label",dataset_name="tasksource/logiqa-2.0-nli")
 
 _oasst = dict(dataset_name="tasksource/oasst2_dense_flat",
     pre_process = lambda ds:ds.filter(lambda x:x['lang']=='en'))
@@ -907,7 +909,6 @@ oasst1__quality = Classification("parent_text","text",labels="quality",**_oasst)
 oasst1__toxicity = Classification("parent_text","text",labels="toxicity",**_oasst)
 oasst1__helpfulness = Classification("parent_text","text",labels="helpfulness",**_oasst)
 
-
 mindgames = Classification("premise","hypothesis","label",dataset_name="sileod/mindgames")
 
 def _udep_post_process(ds):
@@ -956,6 +957,7 @@ winowhy = Classification('sentence', lambda x: f'In "{x["wnli_sent1"]}", {x["wnl
 #for CFG in "cognitive-bias", "fake-news", "gender-bias", "hate-speech", "linguistic-bias", "political-bias", "racial-bias", "text-level-bias":
 #    print(f"mbib__{CFG.replace('-','_')} = Classification('text',labels=name('label',['not {CFG}','{CFG}']), dataset_name='mediabiasgroup/mbib-base', config_name='{CFG}')")
 
+"""
 mbib_cognitive_bias	= Classification('text',labels=name('label',['not cognitive-bias','cognitive-bias']), dataset_name='mediabiasgroup/mbib-base', config_name='cognitive-bias')
 mbib_fake_news	= Classification('text',labels=name('label',['not fake-news','fake-news']), dataset_name='mediabiasgroup/mbib-base', config_name='fake-news')
 mbib_gender_bias	= Classification('text',labels=name('label',['not gender-bias','gender-bias']), dataset_name='mediabiasgroup/mbib-base', config_name='gender-bias')
@@ -964,6 +966,7 @@ mbib_linguistic_bias	= Classification('text',labels=name('label',['not linguisti
 mbib_political_bias	= Classification('text',labels=name('label',['not political-bias','political-bias']), dataset_name='mediabiasgroup/mbib-base', config_name='political-bias')
 mbib_racial_bias	= Classification('text',labels=name('label',['not racial-bias','racial-bias']), dataset_name='mediabiasgroup/mbib-base', config_name='racial-bias')
 mbib_text_level_bias	= Classification('text',labels=name('label',['not text-level-bias','text-level-bias']), dataset_name='mediabiasgroup/mbib-base', config_name='text-level-bias')
+"""
 
 robustLR = Classification("context","statement","label", dataset_name="tasksource/robustLR")
 
@@ -1026,7 +1029,7 @@ def _icl_rand(x):
 icl = Classification("inputs", lambda x: x['symbols'][_icl_rand(x)],
     labels=lambda x: str(x['symbols'][_icl_rand(x)]==x['targets']),
     dataset_name="tasksource/icl-symbol-tuning-instruct",
-    pre_process=lambda ds:ds.filter(lambda x:len(x['inputs'])<200*4), # 200 tokens of 4 char 
+    pre_process=lambda ds:ds.filter(lambda x:len(x['inputs'])<500*4), # 500 tokens of 4 char 
 )
 
 space_nli = Classification("premises","hypothesis","label",dataset_name="tasksource/SpaceNLI")
@@ -1144,9 +1147,11 @@ nope = Classification('premise','hypothesis',
 
 logicNLI = Classification('premise','hypothesis','label',dataset_name='tasksource/LogicNLI')
 
-contract_nli = Classification("premise","hypothesis","label", dataset_name="kiddothe2b/contract-nli",config_name="contractnli_a")
+contract_nli__seg = Classification("premise","hypothesis","label", dataset_name="kiddothe2b/contract-nli",config_name="contractnli_a")
 
-nli4ct = Classification(lambda x: "\n".join(x['Primary_evidence']),'Statement',
+contract_nli__full = Classification("premise","hypothesis","label", dataset_name="kiddothe2b/contract-nli",config_name="contractnli_b")
+
+nli4ct = Classification(lambda x: "\n".join(x['Primary_evidence']),'Statement',"Label",
     dataset_name="AshtonIsNotHere/nli4ct_semeval2024",splits=['train','dev',None])
 
 lsat_ar = MultipleChoice(
@@ -1196,16 +1201,78 @@ ambigNQ = Classification("question",labels=lambda x:{True:"ambiguous", False:"no
 
 siga_nli = Classification("premise","statement","label",dataset_name="tasksource/SIGA-nli")
 
-#unigram_fol = Classification("premise","hypothesis","label",dataset_name='unigram/fol-01')
+unigram_fol = Classification("premise","hypothesis","label",dataset_name='unigram/FOL-nli')
 
-gs_goal = MultipleChoice("sent2",regen("ending[0-3]"),"label",
-        dataset_name="tasksource/goal-step-wikihow",config_name="goal")
+#gs_goal = MultipleChoice("sent2",regen("ending[0-3]"),"label",
+#        dataset_name="tasksource/goal-step-wikihow",config_name="goal")
 
-gs_step = MultipleChoice("sent2",regen("ending[0-3]"),"label",
-        dataset_name="tasksource/goal-step-wikihow",config_name="step")
+#gs_step = MultipleChoice("sent2",regen("ending[0-3]"),"label",
+#        dataset_name="tasksource/goal-step-wikihow",config_name="step")
 
 gs_order = MultipleChoice("sent2",regen("ending[0-1]"),"label",
         dataset_name="tasksource/goal-step-wikihow",config_name="order")
 
-paradise = MultipleChoice("sent2",regen("ending[0-1]"),"label",
+paradise = MultipleChoice("sent2",regen("ending[0-3]"),"label",
       dataset_name="GGLab/PARADISE")
+
+docnli = Classification("premise","hypothesis","label",dataset_name="tasksource/doc-nli")
+
+mctest_nli = Classification("premise","hypothesis","label",dataset_name="tasksource/mctest-nli")
+
+patent_phrase_similarity = Classification("anchor","target","label",dataset_name="tasksource/patent-phrase-similarity")
+
+nlsat = Classification('sentence',labels='label',dataset_name="tasksource/natural-language-satisfiability")
+
+idioms_nli = Classification('premise','hypothesis','label',dataset_name="tasksource/idioms-nli")
+
+lifeycle_entailment = Classification("premise","hypothesis","label",dataset_name='tasksource/lifecycle-entailment')
+
+
+helpsteer__helpfulness = Classification("prompt", "response", "helpfulness", dataset_name="nvidia/HelpSteer")
+helpsteer__correctness = Classification("prompt", "response", "correctness", dataset_name="nvidia/HelpSteer")
+helpsteer__coherence = Classification("prompt", "response", "coherence", dataset_name="nvidia/HelpSteer")
+helpsteer__complexity = Classification("prompt", "response", "complexity", dataset_name="nvidia/HelpSteer")
+helpsteer__verbosity = Classification("prompt", "response", "verbosity", dataset_name="nvidia/HelpSteer")
+
+helpsteer_2__helpfulness = Classification("prompt","response","helpfulness",dataset_name="nvidia/HelpSteer2")
+helpsteer_2__correctness = Classification("prompt", "response", "correctness", dataset_name="nvidia/HelpSteer2")
+helpsteer_2__coherence = Classification("prompt", "response", "coherence", dataset_name="nvidia/HelpSteer2")
+helpsteer_2__complexity = Classification("prompt", "response", "complexity", dataset_name="nvidia/HelpSteer2")
+helpsteer_2__verbosity = Classification("prompt", "response", "verbosity", dataset_name="nvidia/HelpSteer2")
+
+msci_nli = Classification('sentence1','sentence2','label',dataset_name='sadat2307/MSciNLI')
+
+#lex_glue___ecthr_a = Classification(sentence1="text", labels="labels",dataset_name="coastalcph/lex_glue",config_name="ecthr_a") # too long
+#lex_glue___ecthr_b = Classification(sentence1="text", labels="labels") # too long
+
+ultrafeedback = MultipleChoice("question", choices=['response_j','response_k'],labels=constant(0), dataset_name="pushpdeep/UltraFeedback-paired")
+
+essay_scoring = Classification("full_text",labels="score",dataset_name='tasksource/AES2-essay-scoring')
+
+#argument_feedback = Classification("discourse_text",labels="discourse_effectiveness", dataset_name="tasksource/argument-feedback")
+
+eg = lambda x: Classification("full_text", labels=lambda y:int(y[x]), dataset_name="tasksource/english-grading")
+grading__cohesion = eg('cohesion')
+grading__syntax = eg('syntax')
+grading__vocabulary = eg('vocabulary')
+grading__phraseology = eg('phraseology')
+grading__grammar = eg('grammar')
+grading__conventions = eg('conventions')
+
+wice = Classification(lambda x: "\n".join(x['evidence']),'claim','label',
+    dataset_name='tasksource/wice')
+
+hover = Classification("evidence","claim","label",
+    dataset_name="Dzeniks/hover") 
+
+hover__nli = Classification("evidence","claim",name("label",["entailment","neutral","contradiction"]),
+    dataset_name="Dzeniks/hover-3way")
+
+tasksource_dpo = MultipleChoice("prompt",choices=['chosen','rejected'],labels=constant(0),
+    dataset_name="tasksource/tasksource_dpo_pairs")
+
+seahorse = Classification('article',cat(["summary", "question"]),'answer',
+    dataset_name="tasksource/seahorse_summarization_evaluation")
+
+mip = Classification("prompt",labels="y",
+    dataset_name="sileod/missing-item-prediction",config_name="contrastive")