tasksource 0.0.43__tar.gz → 0.0.45__tar.gz

{tasksource-0.0.43 → tasksource-0.0.45}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tasksource
-Version: 0.0.43
+Version: 0.0.45
 Summary: Preprocessings to prepare datasets for a task
 Home-page: https://github.com/sileod/tasksource/
 Classifier: Programming Language :: Python :: 3

{tasksource-0.0.43 → tasksource-0.0.45}/src/tasksource/.ipynb_checkpoints/tasks-checkpoint.py

@@ -38,8 +38,8 @@ anli__a3 = Classification('premise','hypothesis','label', splits=['train_r3','de
 
 
 babi_nli = Classification("premise", "hypothesis", "label",
-    dataset_name="metaeval/babi_nli",
-    config_name=set(get_dataset_config_names("metaeval/babi_nli"))-{"agents-motivations"}
+    dataset_name="tasksource/babi_nli",
+    config_name=set(get_dataset_config_names("tasksource/babi_nli"))-{"agents-motivations"}
 ) # agents-motivations task is not as clear-cut as the others
 
 
@@ -60,7 +60,7 @@ hans = Classification(sentence1="premise", sentence2="hypothesis", labels="label
 
 wanli = Classification('premise','hypothesis','gold', dataset_name="alisawuffles/WANLI")
 
-recast_nli = Classification(sentence1="context", sentence2="hypothesis", labels="label", dataset_name="metaeval/recast",
+recast_nli = Classification(sentence1="context", sentence2="hypothesis", labels="label", dataset_name="tasksource/recast",
     config_name=['recast_kg_relations', 'recast_puns', 'recast_factuality', 'recast_verbnet',
     'recast_verbcorner', 'recast_ner', 'recast_sentiment', 'recast_megaveridicality'])
 
@@ -153,15 +153,15 @@ def _imppres_post_process(ds,prefix=''):
     names=[f'{prefix}_entailment',f'{prefix}_neutral',f'{prefix}_contradiction']))
 
 imppres__presupposition = imppres__prag = Classification("premise","hypothesis","gold_label",
-    dataset_name="metaeval/imppres", config_name=imppres_presupposition,
+    dataset_name="tasksource/imppres", config_name=imppres_presupposition,
     post_process=_imppres_post_process)
 
 imppres__prag = Classification("premise","hypothesis","gold_label_prag",
-    dataset_name="metaeval/imppres", config_name=imppres_implicature,
+    dataset_name="tasksource/imppres", config_name=imppres_implicature,
     post_process=lambda x: _imppres_post_process(x,'pragmatic'))
 
 imppres__log = Classification("premise","hypothesis","gold_label_log",
-    dataset_name="metaeval/imppres", config_name=imppres_implicature,
+    dataset_name="tasksource/imppres", config_name=imppres_implicature,
     post_process=lambda x: _imppres_post_process(x,'logical'))
 
 
@@ -212,6 +212,7 @@ bigbench = MultipleChoice(
     dataset_name='tasksource/bigbench',
     config_name=bigbench_discriminative_english - {"social_i_qa","intersect_geometry"} # english multiple choice tasks, minus duplicates
 )
+#"goal_step_wikihow"
 
 blimp_hard = MultipleChoice(inputs=constant(''),
     choices=['sentence_good','sentence_bad'],
@@ -426,8 +427,6 @@ silicone = Classification("Utterance",labels="Label",
     config_name=['dyda_da', 'dyda_e', 'iemocap', 'maptask', 'meld_e', 'meld_s', 'oasis', 'sem'] # +['swda', 'mrda'] # in pragmeval
 )
 
-#lex_glue___ecthr_a = Classification(sentence1="text", labels="labels") # too long
-#lex_glue___ecthr_b = Classification(sentence1="text", labels="labels") # too long
 lex_glue___eurlex = Classification(sentence1="text", labels="labels") 
 lex_glue___scotus = Classification(sentence1="text", labels="label")
 lex_glue___ledgar = Classification(sentence1="text", labels="label")
@@ -508,7 +507,7 @@ relbert_lexical_relation_classification = Classification(sentence1="head", sente
  config_name=["BLESS","CogALexV","EVALution","K&H+N","ROOT09"])
 
 
-metaeval_linguisticprobing = Classification("sentence", labels="label", dataset_name="metaeval/linguisticprobing", 
+linguisticprobing = Classification("sentence", labels="label", dataset_name="tasksource/linguisticprobing", 
     config_name=['subj_number',
                 'obj_number',
                 'past_present',
@@ -677,9 +676,9 @@ logiqa = MultipleChoice(
 wiki_qa = Classification("question","answer", name("label",['False','True']))
 
 cycic_classification = Classification("question",labels=name("correct_answer",['False','True']),
-    dataset_name = "metaeval/cycic_classification")
+    dataset_name = "tasksource/cycic_classification")
 cycic_mc = MultipleChoice("question", choices=regen('answer\_option[0-4]'), labels="correct_answer",
-    dataset_name = "metaeval/cycic_multiplechoice")
+    dataset_name = "tasksource/cycic_multiplechoice")
 
 
 def _preprocess_chatgpt_detection(ex):
@@ -694,24 +693,24 @@ def _preprocess_chatgpt_detection(ex):
 #    pre_process=lambda dataset:dataset.map(_preprocess_chatgpt_detection))
 
 sts_companion = Classification("sentence1","sentence2","label",
-    dataset_name="metaeval/sts-companion")
+    dataset_name="tasksource/sts-companion")
 
 commonsense_qa_2 = Classification("question",labels="answer",
-    dataset_name="metaeval/commonsense_qa_2.0")
+    dataset_name="tasksource/commonsense_qa_2.0")
 
-ling_nli = Classification("premise","hypothesis","label",dataset_name="metaeval/lingnli")
+ling_nli = Classification("premise","hypothesis","label",dataset_name="tasksource/lingnli")
 
 monotonicity_entailment = Classification("sentence1", "sentence2", "gold_label",    
-    dataset_name="metaeval/monotonicity-entailment")
+    dataset_name="tasksource/monotonicity-entailment")
 
 arct = MultipleChoice(cat(["reason","claim"]),choices=["warrant0","warrant1"],
-    labels="correctLabelW0orW1", dataset_name="metaeval/arct")
+    labels="correctLabelW0orW1", dataset_name="tasksource/arct")
 
 scinli = Classification("sentence1", "sentence2", labels="label",
     post_process=lambda x:x.shuffle(seed=0),
-    dataset_name="metaeval/scinli")
+    dataset_name="tasksource/scinli")
 
-naturallogic = Classification(" sent1 "," sent2 "," new_label ",dataset_name="metaeval/naturallogic")
+naturallogic = Classification(" sent1 "," sent2 "," new_label ",dataset_name="tasksource/naturallogic")
 
 onestop_qa = MultipleChoice(cat(["paragraph","question"]),choices_list="answers",
     labels=constant(0))
@@ -727,7 +726,7 @@ dyna_hate = Classification("text",labels="label",dataset_name="aps/dynahate",spl
 
 syntactic_augmentation_nli = Classification('sentence1',"sentence2","gold_label",dataset_name="metaeval/syntactic-augmentation-nli")
 
-autotnli = Classification("premises", "hypothesis", "label", dataset_name="metaeval/autotnli")
+autotnli = Classification("premises", "hypothesis", "label", dataset_name="tasksource/autotnli")
 #equate = Classification("sentence1", "sentence2", "gold_label",dataset_name="metaeval/equate")
 
 conqada = Classification("sentence1","sentence2","label",dataset_name="lasha-nlp/CONDAQA",
@@ -747,9 +746,9 @@ wouldyourather = MultipleChoice(constant('Most people would rather:'), choices=[
     labels= lambda x: int(x['votes_a']<x['votes_b']),
     dataset_name="metaeval/wouldyourather")
 
-attempto_nli = Classification("premise","hypothesis",
-    lambda x:f'race-{x["race_label"]}',
-    dataset_name="sileod/attempto-nli")
+#attempto_nli = Classification("premise","hypothesis",
+#    lambda x:f'race-{x["race_label"]}',
+#    dataset_name="sileod/attempto-nli")
 
 defeasible_nli = Classification(cat(["Premise","Hypothesis"]),"Update",labels="UpdateType",
     dataset_name="metaeval/defeasible-nli",config_name=['atomic', 'snli'])
@@ -758,25 +757,22 @@ defeasible_nli = Classification(cat(["Premise","Hypothesis"]),"Update",labels="U
 #    dataset_name="metaeval/defeasible-nli",config_name='social')
 
 help_nli = Classification("ori_sentence","new_sentence","gold_label",
-    dataset_name="metaeval/help-nli")
+    dataset_name="tasksource/help-nli")
     
 nli_veridicality_transitivity = Classification("sentence1","sentence2","gold_label",
     dataset_name="metaeval/nli-veridicality-transitivity")
 
-nl_satisfiability= Classification("sentence",labels="label",
-    dataset_name="metaeval/natural-language-satisfiability")
-
 lonli = Classification("premise","hypothesis","label",
-    dataset_name="metaeval/lonli")
+    dataset_name="tasksource/lonli")
 
 dadc_limit = Classification("sentence1","sentence2","label",
-    dataset_name="metaeval/dadc-limit-nli")
+    dataset_name="tasksource/dadc-limit-nli")
 
 flute = Classification("premise","hypothesis","label",
     dataset_name="ColumbiaNLP/FLUTE")
 
 strategy_qa = Classification('question',labels='answer',
-    dataset_name="metaeval/strategy-qa",splits=['train',None,None])
+    dataset_name="tasksource/strategy-qa",splits=['train',None,None])
 
 summarize_from_feedback = MultipleChoice(get.info.post,
     choices_list=lambda x: [x['summaries'][0]['text'],x['summaries'][1]['text']],
@@ -790,10 +786,10 @@ folio = Classification("premises","conclusion",
     dataset_name="tasksource/folio")
 
 tomi_nli = Classification("premise","hypothesis","label",
-    dataset_name="metaeval/tomi-nli")
+    dataset_name="tasksource/tomi-nli")
 
 avicenna = Classification("Premise 1","Premise 2","Syllogistic relation",
-    dataset_name="metaeval/avicenna")
+    dataset_name="tasksource/avicenna")
 
 shp = MultipleChoice("history",
     choices=['human_ref_A','human_ref_B'],
@@ -816,7 +812,7 @@ creak = Classification("sentence",labels="label",
 
 mutual = MultipleChoice("article",choices_list="options",
     labels=lambda x: "ABCD".index(x['answers']),
-    dataset_name="metaeval/mutual",splits=["train",None,None])
+    dataset_name="tasksource/mutual",splits=["train",None,None])
 
 neqa = MultipleChoice('prompt',choices_list='classes',labels="answer_index",
     dataset_name="inverse-scaling/NeQA")
@@ -826,35 +822,35 @@ redefine_math = MultipleChoice('prompt',choices_list='classes',labels="answer_in
     dataset_name="inverse-scaling/redefine-math")
 
 puzzte = Classification("puzzle_text","question","answer",
-    dataset_name="metaeval/puzzte")
+    dataset_name="tasksource/puzzte")
 
 implicatures = MultipleChoice(cat(['context','response'],"\n"),
     choices=['correct_implicature','incorrect_implicature'],
     labels=constant(0),
-    dataset_name='metaeval/implicatures')
+    dataset_name='tasksource/implicatures')
 
 race = MultipleChoice(cat(['question','article'],'\n'), choices_list='options',
     labels=lambda x:'ABCDE'.index(x['answer']),
     config_name=['middle','high'])
 
 race_c = MultipleChoice(cat(['question','article'],'\n'),choices_list='option',labels='label',
-    dataset_name='metaeval/race-c')
+    dataset_name='tasksource/race-c')
 
 spartqa_yn=Classification("story","question","answer",
-    dataset_name="metaeval/spartqa-yn")
+    dataset_name="tasksource/spartqa-yn")
 
 spartqa_mc=MultipleChoice(cat(["story","question"]),choices_list="candidate_answers",labels="answer",
-    dataset_name="metaeval/spartqa-mchoice")
+    dataset_name="tasksource/spartqa-mchoice")
 
 temporal_nli = Classification("Premise","Hypothesis","Label",
-    dataset_name="metaeval/temporal-nli")
+    dataset_name="tasksource/temporal-nli")
 
 riddle_sense = MultipleChoice("question", choices_list=get.choices.text, 
     labels=lambda x : "ABCDE".index(x['answerKey']))
 
 clcd = Classification(
     "sentence1","sentence2","label",
-    dataset_name="metaeval/clcd-english")
+    dataset_name="tasksource/clcd-english")
 
 twentyquestions = Classification("question","subject","answer",dataset_name="maximedb/twentyquestions")
 
@@ -862,16 +858,16 @@ reclor = MultipleChoice(cat(["context","question"]),choices_list="answers",label
     dataset_name="metaeval/reclor",splits=['train','validation',None])
 
 c_aug_imdb = Classification("Text",labels="Sentiment",
-    dataset_name='metaeval/counterfactually-augmented-imdb')
+    dataset_name='tasksource/counterfactually-augmented-imdb')
 
 c_aug_snli = Classification("sentence1","sentence2","gold_label",
-    dataset_name='metaeval/counterfactually-augmented-snli')
+    dataset_name='tasksource/counterfactually-augmented-snli')
 
 cnli = Classification("premise","hypothesis","label",
     dataset_name='metaeval/cnli')
 
 perturbed_boolq = Classification("question",labels="hard_label",
-    dataset_name='metaeval/boolq-natural-perturbations')
+    dataset_name='tasksource/boolq-natural-perturbations')
 
 #mega_acceptability = Classification("sentence",labels="average",
 #    dataset_name='metaeval/mega-acceptability-v2')
@@ -883,14 +879,14 @@ equate = Classification("sentence1","sentence2","gold_label",
     dataset_name='metaeval/equate')
 
 science_qa = MultipleChoice("question",choices_list="choices",labels="answer",
-    dataset_name="metaeval/ScienceQA_text_only")
+    dataset_name="tasksource/ScienceQA_text_only")
 
 ekar=MultipleChoice("question",choices_list=get.choices.text,
     labels=lambda x:"ABCD".index(x['answerKey']),
 dataset_name="Jiangjie/ekar_english")
 
 implicit_hate = Classification("post",labels="class",
-    dataset_name="metaeval/implicit-hate-stg1")
+    dataset_name="tasksource/implicit-hate-stg1")
 
 nli_unambiguity = Classification("premise","hypothesis","gini",
     dataset_name="metaeval/chaos-mnli-ambiguity")
@@ -898,7 +894,7 @@ nli_unambiguity = Classification("premise","hypothesis","gini",
 headline_cause = Classification('left_title','right_title','label',
     dataset_name='IlyaGusev/headline_cause',config_name='en_simple')
 
-logiqa_2 = Classification("premise","hypothesis","label",dataset_name="metaeval/logiqa-2.0-nli")
+logiqa_2 = Classification("premise","hypothesis","label",dataset_name="tasksource/logiqa-2.0-nli")
 
 _oasst = dict(dataset_name="tasksource/oasst2_dense_flat",
     pre_process = lambda ds:ds.filter(lambda x:x['lang']=='en'))
@@ -907,7 +903,6 @@ oasst1__quality = Classification("parent_text","text",labels="quality",**_oasst)
 oasst1__toxicity = Classification("parent_text","text",labels="toxicity",**_oasst)
 oasst1__helpfulness = Classification("parent_text","text",labels="helpfulness",**_oasst)
 
-
 mindgames = Classification("premise","hypothesis","label",dataset_name="sileod/mindgames")
 
 def _udep_post_process(ds):
@@ -956,6 +951,7 @@ winowhy = Classification('sentence', lambda x: f'In "{x["wnli_sent1"]}", {x["wnl
 #for CFG in "cognitive-bias", "fake-news", "gender-bias", "hate-speech", "linguistic-bias", "political-bias", "racial-bias", "text-level-bias":
 #    print(f"mbib__{CFG.replace('-','_')} = Classification('text',labels=name('label',['not {CFG}','{CFG}']), dataset_name='mediabiasgroup/mbib-base', config_name='{CFG}')")
 
+"""
 mbib_cognitive_bias	= Classification('text',labels=name('label',['not cognitive-bias','cognitive-bias']), dataset_name='mediabiasgroup/mbib-base', config_name='cognitive-bias')
 mbib_fake_news	= Classification('text',labels=name('label',['not fake-news','fake-news']), dataset_name='mediabiasgroup/mbib-base', config_name='fake-news')
 mbib_gender_bias	= Classification('text',labels=name('label',['not gender-bias','gender-bias']), dataset_name='mediabiasgroup/mbib-base', config_name='gender-bias')
@@ -964,6 +960,7 @@ mbib_linguistic_bias	= Classification('text',labels=name('label',['not linguisti
 mbib_political_bias	= Classification('text',labels=name('label',['not political-bias','political-bias']), dataset_name='mediabiasgroup/mbib-base', config_name='political-bias')
 mbib_racial_bias	= Classification('text',labels=name('label',['not racial-bias','racial-bias']), dataset_name='mediabiasgroup/mbib-base', config_name='racial-bias')
 mbib_text_level_bias	= Classification('text',labels=name('label',['not text-level-bias','text-level-bias']), dataset_name='mediabiasgroup/mbib-base', config_name='text-level-bias')
+"""
 
 robustLR = Classification("context","statement","label", dataset_name="tasksource/robustLR")
 
@@ -1026,7 +1023,7 @@ def _icl_rand(x):
 icl = Classification("inputs", lambda x: x['symbols'][_icl_rand(x)],
     labels=lambda x: str(x['symbols'][_icl_rand(x)]==x['targets']),
     dataset_name="tasksource/icl-symbol-tuning-instruct",
-    pre_process=lambda ds:ds.filter(lambda x:len(x['inputs'])<200*4), # 200 tokens of 4 char 
+    pre_process=lambda ds:ds.filter(lambda x:len(x['inputs'])<500*4), # 500 tokens of 4 char 
 )
 
 space_nli = Classification("premises","hypothesis","label",dataset_name="tasksource/SpaceNLI")
@@ -1196,16 +1193,59 @@ ambigNQ = Classification("question",labels=lambda x:{True:"ambiguous", False:"no
 
 siga_nli = Classification("premise","statement","label",dataset_name="tasksource/SIGA-nli")
 
-#unigram_fol = Classification("premise","hypothesis","label",dataset_name='unigram/fol-01')
+unigram_fol = Classification("premise","hypothesis","label",dataset_name='unigram/FOL-nli')
 
-gs_goal = MultipleChoice("sent2",regen("ending[0-3]"),"label",
-        dataset_name="tasksource/goal-step-wikihow",config_name="goal")
+#gs_goal = MultipleChoice("sent2",regen("ending[0-3]"),"label",
+#        dataset_name="tasksource/goal-step-wikihow",config_name="goal")
 
-gs_step = MultipleChoice("sent2",regen("ending[0-3]"),"label",
-        dataset_name="tasksource/goal-step-wikihow",config_name="step")
+#gs_step = MultipleChoice("sent2",regen("ending[0-3]"),"label",
+#        dataset_name="tasksource/goal-step-wikihow",config_name="step")
 
 gs_order = MultipleChoice("sent2",regen("ending[0-1]"),"label",
         dataset_name="tasksource/goal-step-wikihow",config_name="order")
 
 paradise = MultipleChoice("sent2",regen("ending[0-1]"),"label",
       dataset_name="GGLab/PARADISE")
+
+docnli = Classification("premise","hypothesis","label",dataset_name="tasksource/doc-nli")
+
+mctest_nli = Classification("premise","hypothesis","label",dataset_name="tasksource/mctest-nli")
+
+patent_phrase_similarity = Classification("anchor","target","label",dataset_name="tasksource/patent-phrase-similarity")
+
+nlsat = Classification('sentence',labels='label',dataset_name="tasksource/natural-language-satisfiability")
+
+idioms_nli = Classification('premise','hypothesis','label',dataset_name="tasksource/idioms-nli")
+
+lifeycle_entailment = Classification("premise","hypothesis","label",dataset_name='tasksource/lifecycle-entailment')
+
+helpsteer_2__helpfulness = Classification("prompt","response","helpfulness",dataset_name="nvidia/HelpSteer2")
+helpsteer_2__correctness = Classification("prompt", "response", "correctness", dataset_name="nvidia/HelpSteer2")
+helpsteer_2__coherence = Classification("prompt", "response", "coherence", dataset_name="nvidia/HelpSteer2")
+helpsteer_2__complexity = Classification("prompt", "response", "complexity", dataset_name="nvidia/HelpSteer2")
+helpsteer_2__verbosity = Classification("prompt", "response", "verbosity", dataset_name="nvidia/HelpSteer2")
+
+helpsteer__helpfulness = Classification("prompt", "response", "helpfulness", dataset_name="nvidia/HelpSteer")
+helpsteer__correctness = Classification("prompt", "response", "correctness", dataset_name="nvidia/HelpSteer")
+helpsteer__coherence = Classification("prompt", "response", "coherence", dataset_name="nvidia/HelpSteer")
+helpsteer__complexity = Classification("prompt", "response", "complexity", dataset_name="nvidia/HelpSteer")
+helpsteer__verbosity = Classification("prompt", "response", "verbosity", dataset_name="nvidia/HelpSteer")
+
+msci_nli = Classification('sentence1','sentence2','label',dataset_name='sadat2307/MSciNLI')
+
+lex_glue___ecthr_a = Classification(sentence1="text", labels="labels",dataset_name="coastalcph/lex_glue",config_name="ecthr_a") # too long
+lex_glue___ecthr_b = Classification(sentence1="text", labels="labels") # too long
+
+ultrafeedback = MultipleChoice("question", choices=['response_j','response_k'],labels=constant(0), dataset_name="pushpdeep/UltraFeedback-paired")
+
+essay_scoring = Classification("full_text",labels="score",dataset_name='tasksource/AES2-essay-scoring')
+
+argument_feedback = Classification("discourse_text",labels="discourse_effectiveness", dataset_name="tasksource/argument-feedback")
+
+eg = lambda x: Classification("full_text", labels=lambda y:int(y[x]), dataset_name="tasksource/english-grading")
+grading__cohesion = eg('cohesion')
+grading__syntax = eg('syntax')
+grading__vocabulary = eg('vocabulary')
+grading__phraseology = eg('phraseology')
+grading__grammar = eg('grammar')
+grading__conventions = eg('conventions')