dslighting 1.3.9__py3-none-any.whl → 1.7.6__py3-none-any.whl

dslighting/registry/freesound-audio-tagging-2019/grade.py

@@ -0,0 +1,64 @@
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+import sklearn.metrics
+
+from mlebench.grade_helpers import InvalidSubmissionError
+
+from .vocabulary import CLASS_NAMES
+
+
+# Using the metric implementation given to participants. Source:
+# https://colab.research.google.com/drive/1AgPdhSp7ttY18O3fEoHOQKlt_3HJDLi8
+def calculate_overall_lwlrap_sklearn(truth: np.ndarray, scores: np.ndarray) -> float:
+    """Calculate the overall lwlrap using sklearn.metrics.lrap."""
+    # sklearn doesn't correctly apply weighting to samples with no labels, so just skip them.
+    sample_weight = np.sum(truth > 0, axis=1)
+    nonzero_weight_sample_indices = np.flatnonzero(sample_weight > 0)
+    overall_lwlrap = sklearn.metrics.label_ranking_average_precision_score(
+        truth[nonzero_weight_sample_indices, :] > 0,
+        scores[nonzero_weight_sample_indices, :],
+        sample_weight=sample_weight[nonzero_weight_sample_indices],
+    )
+    return overall_lwlrap
+
+
+def prepare_for_metric(submission: pd.DataFrame, answers: pd.DataFrame) -> tuple:
+    """
+    `submission` and `answers` are both DataFrames with N rows and M + 1 columns.
+    The first column is "fname" and the remaining M columns are the predictions for each class.
+    The values are floats between 0 and 1. For `answers`, the values are binary.
+    """
+    if len(submission) != len(answers):
+        raise InvalidSubmissionError("Submission and answers must have the same length")
+
+    if "fname" != submission.columns[0]:
+        raise InvalidSubmissionError("Submission must have an 'fname' column")
+
+    if not all(submission.columns[1:] == CLASS_NAMES):
+        raise InvalidSubmissionError("Submission classes do not match expected classes")
+
+    assert answers.columns[0] == "fname", "Answers must have an 'fname' column"
+    assert all(answers.columns[1:] == CLASS_NAMES), "Answers classes do not match expected classes"
+
+    submission = submission.sort_values("fname").reset_index(drop=True)
+    answers = answers.sort_values("fname").reset_index(drop=True)
+
+    if (submission["fname"].values != answers["fname"].values).any():
+        raise InvalidSubmissionError("Submission and answers must have the same ids")
+
+    # Convert to numbers
+    try:
+        submission.iloc[:, 1:] = submission.iloc[:, 1:].apply(pd.to_numeric)
+    except (ValueError, TypeError) as e:
+        raise InvalidSubmissionError(f"Error converting submission values to float: {str(e)}")
+    answers.iloc[:, 1:] = answers.iloc[:, 1:].apply(pd.to_numeric)
+
+    # Return as numpy arrays
+    return answers.iloc[:, 1:].values, submission.iloc[:, 1:].values
+
+
+def grade(submission: pd.DataFrame, answers: pd.DataFrame) -> float:
+    sub, ans = prepare_for_metric(submission, answers)
+    return calculate_overall_lwlrap_sklearn(sub, ans)

dslighting/registry/freesound-audio-tagging-2019/prepare.py

@@ -0,0 +1,94 @@
+import shutil
+from pathlib import Path
+
+import pandas as pd
+import requests
+
+from mlebench.utils import get_logger
+
+from .vocabulary import CLASS_NAMES
+
+logger = get_logger(__name__)
+
+
+def download_file(url, local_filename):
+    with requests.get(url, stream=True) as r:
+        r.raise_for_status()
+        with open(local_filename, "wb") as f:
+            for chunk in r.iter_content(chunk_size=8192):
+                if chunk:  # filter out keep-alive new chunks
+                    f.write(chunk)
+    return local_filename
+
+
+def prepare(raw: Path, public: Path, private: Path):
+    """
+    Straightforward: we have access to the post-competition released test labels, so we don't need
+    to create our own split here. We just copy over the raw data provided by the competition and
+    download the test labels.
+
+    Otherwise, the only work here is to convert the test data into the right format for grading:
+    The original form of `test.csv` is a DataFrame with N rows and 2 columns. The first column is
+    "fname" and the second column is the labels as comma-separated strings (class names). We
+    convert the test labels into a binary many-hot matrix matching the shape of the submission,
+    [N rows, M + 1 columns]: The first column is "fname" and the remaining M columns are the
+    predictions for each class.
+    """
+
+    # Copy over everything in the raw directory
+    logger.info("Copying raw data to public directory")
+    # Don't copy the metadata file if it exists
+    items_to_copy = [item for item in raw.iterdir() if "FSDKaggle2019.meta" not in item.name]
+    for item in items_to_copy:
+        dest = public / item.name
+        if item.is_dir():
+            shutil.copytree(item, dest)
+        else:
+            shutil.copy(item, dest)
+    assert len(list(public.iterdir())) == len(
+        items_to_copy
+    ), "Expected all files in raw to be copied to public"
+
+    # Download the test labels and metadata that were released after the competition
+    test_url = "https://zenodo.org/records/3612637/files/FSDKaggle2019.meta.zip?download=1"
+    dest_path = raw / "FSDKaggle2019.meta.zip"
+    download_file(test_url, dest_path)
+    logger.info(f"Downloaded file saved as {dest_path}")
+    # # Unzip
+    shutil.unpack_archive(dest_path, raw)
+    unzipped_path = raw / "FSDKaggle2019.meta"
+    logger.info(f"Unzipped file to {unzipped_path}")
+
+    # Read test labels
+    test_post_competition = pd.read_csv(unzipped_path / "test_post_competition.csv")
+    private_test = test_post_competition[test_post_competition["usage"] == "Private"]
+    # Create a binary many-hot matrix
+    new_test_rows = []
+    for idx, row in private_test.iterrows():
+        fname = row["fname"]
+        labels = row["labels"].split(",")
+        labels = [1 if label in labels else 0 for label in CLASS_NAMES]
+        new_test_rows.append([fname] + labels)
+    new_test = pd.DataFrame(new_test_rows, columns=["fname"] + CLASS_NAMES)
+    new_test.to_csv(private / "test.csv", index=False)
+
+    # Check that test and submission match
+    submission = pd.read_csv(public / "sample_submission.csv")
+    assert len(submission) == len(
+        new_test
+    ), f"Expected {len(new_test)} rows in test.csv, but got {len(submission)}"
+    assert (
+        submission.columns[1:].tolist() == CLASS_NAMES
+    ), "Expected class names to match between test.csv and sample_submission.csv"
+    assert all(
+        submission.columns == new_test.columns
+    ), "Expected columns to match between test.csv and sample_submission.csv"
+    new_test.sort_values("fname", inplace=True)
+    submission.sort_values("fname", inplace=True)
+    assert (
+        submission["fname"].tolist() == new_test["fname"].tolist()
+    ), "Expected 'fname' to match between test.csv and sample_submission.csv"
+
+    # Remove the downloaded metadata
+    dest_path.unlink()
+    shutil.rmtree(unzipped_path)

dslighting/registry/freesound-audio-tagging-2019/prepare_val.py

@@ -0,0 +1,175 @@
+import shutil
+from pathlib import Path
+
+import pandas as pd
+import requests
+from sklearn.model_selection import train_test_split
+
+from mlebench.utils import get_logger
+
+from .vocabulary import CLASS_NAMES
+
+logger = get_logger(__name__)
+
+
+def download_file(url, local_filename):
+    with requests.get(url, stream=True) as r:
+        r.raise_for_status()
+        with open(local_filename, "wb") as f:
+            for chunk in r.iter_content(chunk_size=8192):
+                if chunk:  # filter out keep-alive new chunks
+                    f.write(chunk)
+    return local_filename
+
+
+def _create_many_hot_labels(df: pd.DataFrame) -> pd.DataFrame:
+    """Converts a DataFrame with comma-separated labels into a many-hot encoded DataFrame."""
+    new_rows = []
+    for _, row in df.iterrows():
+        fname = row["fname"]
+        labels = row["labels"].split(",")
+        labels_one_hot = [1 if class_name in labels else 0 for class_name in CLASS_NAMES]
+        new_rows.append([fname] + labels_one_hot)
+
+    new_df = pd.DataFrame(new_rows, columns=["fname"] + CLASS_NAMES)
+    return new_df
+
+
+def prepare(raw: Path, public: Path, private: Path):
+    """
+    Straightforward: we have access to the post-competition released test labels, so we don't need
+    to create our own split here. We just copy over the raw data provided by the competition and
+    download the test labels.
+
+    Otherwise, the only work here is to convert the test data into the right format for grading:
+    The original form of `test.csv` is a DataFrame with N rows and 2 columns. The first column is
+    "fname" and the second column is the labels as comma-separated strings (class names). We
+    convert the test labels into a binary many-hot matrix matching the shape of the submission,
+    [N rows, M + 1 columns]: The first column is "fname" and the remaining M columns are the
+    predictions for each class.
+
+    This script also creates a new validation set by splitting the original train_curated set,
+    saving the results in `public_val` and `private_val` directories.
+    """
+    # =================================================================
+    # Original Data Preparation (public/ and private/)
+    # =================================================================
+
+    # Copy over everything in the raw directory
+    logger.info("Copying raw data to public directory")
+    # Don't copy the metadata file if it exists
+    items_to_copy = [item for item in raw.iterdir() if "FSDKaggle2019.meta" not in item.name]
+    for item in items_to_copy:
+        dest = public / item.name
+        if dest.exists():
+            continue
+        if item.is_dir():
+            shutil.copytree(item, dest)
+        else:
+            shutil.copy(item, dest)
+    assert len(list(public.iterdir())) >= len(
+        items_to_copy
+    ), "Expected all files in raw to be copied to public"
+
+    # Download the test labels and metadata that were released after the competition
+    test_url = "https://zenodo.org/records/3612637/files/FSDKaggle2019.meta.zip?download=1"
+    dest_path = raw / "FSDKaggle2019.meta.zip"
+    if not dest_path.exists():
+        download_file(test_url, dest_path)
+        logger.info(f"Downloaded file saved as {dest_path}")
+        # # Unzip
+        shutil.unpack_archive(dest_path, raw)
+        logger.info(f"Unzipped file to {raw / 'FSDKaggle2019.meta'}")
+
+    unzipped_path = raw / "FSDKaggle2019.meta"
+
+    # Read test labels
+    test_post_competition = pd.read_csv(unzipped_path / "test_post_competition.csv")
+    private_test_df = test_post_competition[test_post_competition["usage"] == "Private"]
+    # Create a binary many-hot matrix
+    new_test = _create_many_hot_labels(private_test_df)
+    new_test.to_csv(private / "test.csv", index=False)
+
+    # Check that test and submission match
+    submission = pd.read_csv(public / "sample_submission.csv")
+    assert len(submission) == len(
+        new_test
+    ), f"Expected {len(new_test)} rows in test.csv, but got {len(submission)}"
+    assert (
+        submission.columns[1:].tolist() == CLASS_NAMES
+    ), "Expected class names to match between test.csv and sample_submission.csv"
+    assert all(
+        submission.columns == new_test.columns
+    ), "Expected columns to match between test.csv and sample_submission.csv"
+    new_test.sort_values("fname", inplace=True)
+    submission.sort_values("fname", inplace=True)
+    assert (
+        submission["fname"].tolist() == new_test["fname"].tolist()
+    ), "Expected 'fname' to match between test.csv and sample_submission.csv"
+
+    # Remove the downloaded metadata
+    if dest_path.exists():
+        dest_path.unlink()
+    if unzipped_path.exists():
+        shutil.rmtree(unzipped_path)
+
+    # =================================================================
+    # New Validation Set Creation (public_val/ and private_val/)
+    # =================================================================
+    logger.info("Creating new validation set from train_curated.csv")
+
+    # Define paths and create parallel directories for the validation set
+    public_val = public.parent / "public_val"
+    private_val = private.parent / "private_val"
+    public_val.mkdir(exist_ok=True)
+    private_val.mkdir(exist_ok=True)
+
+    # Load original curated training data and final test set (to match size)
+    original_train_df = pd.read_csv(public / "train_curated.csv")
+    original_test_df = pd.read_csv(private / "test.csv")
+
+    # Split the curated training data to create a new train/validation set.
+    # The new validation set ('test_val') will have the same size as the original test set.
+    train_val_df, test_val_df = train_test_split(
+        original_train_df, test_size=len(original_test_df), random_state=42, shuffle=True
+    )
+
+    # --- Populate public_val directory ---
+    logger.info(f"Populating {public_val} with new training and validation data")
+
+    # Save the new, smaller curated training set manifest
+    train_val_df.to_csv(public_val / "train_curated.csv", index=False)
+
+    # To mirror the directory structure, copy the noisy data as-is
+    shutil.copy(public / "train_noisy.csv", public_val / "train_noisy.csv")
+    if (public / "train_noisy").exists():
+        if (public_val / "train_noisy").exists():
+            shutil.rmtree(public_val / "train_noisy")
+        shutil.copytree(public / "train_noisy", public_val / "train_noisy")
+
+    # Create directories for the new audio file splits
+    (public_val / "train_curated").mkdir(exist_ok=True)
+    (public_val / "test").mkdir(exist_ok=True)
+
+    # Copy audio files for the new, smaller training set
+    for fname in train_val_df["fname"]:
+        shutil.copy(public / "train_curated" / fname, public_val / "train_curated" / fname)
+
+    # Copy audio files for the new validation set into its 'test' directory
+    for fname in test_val_df["fname"]:
+        shutil.copy(public / "train_curated" / fname, public_val / "test" / fname)
+
+    # Create a new sample submission file corresponding to the validation set
+    val_submission = pd.DataFrame({"fname": test_val_df["fname"].sort_values()})
+    for col in CLASS_NAMES:
+        val_submission[col] = 0
+    val_submission.to_csv(public_val / "sample_submission.csv", index=False)
+
+    # --- Populate private_val directory ---
+    logger.info(f"Populating {private_val} with new validation labels")
+
+    # Create and save the ground truth labels for the new validation set
+    private_test_val_df = _create_many_hot_labels(test_val_df)
+    private_test_val_df.to_csv(private_val / "test.csv", index=False)
+
+    logger.info("Validation set creation complete.")

dslighting/registry/freesound-audio-tagging-2019/vocabulary.py

@@ -0,0 +1,83 @@
+CLASS_NAMES = [
+    "Accelerating_and_revving_and_vroom",
+    "Accordion",
+    "Acoustic_guitar",
+    "Applause",
+    "Bark",
+    "Bass_drum",
+    "Bass_guitar",
+    "Bathtub_(filling_or_washing)",
+    "Bicycle_bell",
+    "Burping_and_eructation",
+    "Bus",
+    "Buzz",
+    "Car_passing_by",
+    "Cheering",
+    "Chewing_and_mastication",
+    "Child_speech_and_kid_speaking",
+    "Chink_and_clink",
+    "Chirp_and_tweet",
+    "Church_bell",
+    "Clapping",
+    "Computer_keyboard",
+    "Crackle",
+    "Cricket",
+    "Crowd",
+    "Cupboard_open_or_close",
+    "Cutlery_and_silverware",
+    "Dishes_and_pots_and_pans",
+    "Drawer_open_or_close",
+    "Drip",
+    "Electric_guitar",
+    "Fart",
+    "Female_singing",
+    "Female_speech_and_woman_speaking",
+    "Fill_(with_liquid)",
+    "Finger_snapping",
+    "Frying_(food)",
+    "Gasp",
+    "Glockenspiel",
+    "Gong",
+    "Gurgling",
+    "Harmonica",
+    "Hi-hat",
+    "Hiss",
+    "Keys_jangling",
+    "Knock",
+    "Male_singing",
+    "Male_speech_and_man_speaking",
+    "Marimba_and_xylophone",
+    "Mechanical_fan",
+    "Meow",
+    "Microwave_oven",
+    "Motorcycle",
+    "Printer",
+    "Purr",
+    "Race_car_and_auto_racing",
+    "Raindrop",
+    "Run",
+    "Scissors",
+    "Screaming",
+    "Shatter",
+    "Sigh",
+    "Sink_(filling_or_washing)",
+    "Skateboard",
+    "Slam",
+    "Sneeze",
+    "Squeak",
+    "Stream",
+    "Strum",
+    "Tap",
+    "Tick-tock",
+    "Toilet_flush",
+    "Traffic_noise_and_roadway_noise",
+    "Trickle_and_dribble",
+    "Walk_and_footsteps",
+    "Water_tap_and_faucet",
+    "Waves_and_surf",
+    "Whispering",
+    "Writing",
+    "Yell",
+    "Zipper_(clothing)",
+]
+assert len(CLASS_NAMES) == 80, f"Expected 80 class names, but got {len(CLASS_NAMES)}"

dslighting/registry/google-quest-challenge/checksums.yaml

@@ -0,0 +1,7 @@
+private:
+  test.csv: 2bc3cc2878f0c50a05858f6562b12469
+public:
+  sample_submission.csv: ba75781e8b1a5936533c4de82e515c98
+  test.csv: 06f482b597547c3c975ff1fc59314011
+  train.csv: f41aa13133d70b94bfa53801762937aa
+zip: 9b2e21108f8c0f690457ec23c0d7d249

dslighting/registry/google-quest-challenge/classes.py

@@ -0,0 +1,32 @@
+CLASSES = [
+    "question_asker_intent_understanding",
+    "question_body_critical",
+    "question_conversational",
+    "question_expect_short_answer",
+    "question_fact_seeking",
+    "question_has_commonly_accepted_answer",
+    "question_interestingness_others",
+    "question_interestingness_self",
+    "question_multi_intent",
+    "question_not_really_a_question",
+    "question_opinion_seeking",
+    "question_type_choice",
+    "question_type_compare",
+    "question_type_consequence",
+    "question_type_definition",
+    "question_type_entity",
+    "question_type_instructions",
+    "question_type_procedure",
+    "question_type_reason_explanation",
+    "question_type_spelling",
+    "question_well_written",
+    "answer_helpful",
+    "answer_level_of_information",
+    "answer_plausible",
+    "answer_relevance",
+    "answer_satisfaction",
+    "answer_type_instructions",
+    "answer_type_procedure",
+    "answer_type_reason_explanation",
+    "answer_well_written",
+]

dslighting/registry/google-quest-challenge/config.yaml

@@ -0,0 +1,26 @@
+id: google-quest-challenge
+name: Google QUEST Q&A Labeling
+competition_type: code
+awards_medals: true
+prizes:
+  - position: 1
+    value: 10000
+  - position: 2
+    value: 7000
+  - position: 3
+    value: 5000
+  - position: 4
+    value: 2000
+  - position: 5
+    value: 1000
+description: mlebench/competitions/google-quest-challenge/description.md
+
+dataset:
+  answers: google-quest-challenge/prepared/private/test.csv
+  sample_submission: google-quest-challenge/prepared/public/sample_submission.csv
+
+grader:
+  name: column-wise-spearman
+  grade_fn: mlebench.competitions.google-quest-challenge.grade:grade
+
+preparer: mlebench.competitions.google-quest-challenge.prepare:prepare

dslighting/registry/google-quest-challenge/description.md

@@ -0,0 +1,83 @@
+# Overview
+## Description
+Computers are really good at answering questions with single, verifiable answers. But, humans are often still better at answering questions about opinions, recommendations, or personal experiences.
+
+Humans are better at addressing subjective questions that require a deeper, multidimensional understanding of context - something computers aren't trained to do well…yet.. Questions can take many forms - some have multi-sentence elaborations, others may be simple curiosity or a fully developed problem. They can have multiple intents, or seek advice and opinions. Some may be helpful and others interesting. Some are simple right or wrong.
+
+![Human computable dimensions](https://storage.googleapis.com/kaggle-media/competitions/google-research/human_computable_dimensions_1.png)
+
+Unfortunately, it’s hard to build better subjective question-answering algorithms because of a lack of data and predictive models. That’s why the [CrowdSource](https://crowdsource.google.com/csf/#/) team at Google Research, a group dedicated to advancing NLP and other types of ML science via crowdsourcing, has collected data on a number of these quality scoring aspects.
+
+In this competition, you’re challenged to use this new dataset to build predictive algorithms for different subjective aspects of question-answering. The question-answer pairs were gathered from nearly 70 different websites, in a "common-sense" fashion. Our raters received minimal guidance and training, and relied largely on their subjective interpretation of the prompts. As such, each prompt was crafted in the most intuitive fashion so that raters could simply use their common-sense to complete the task. By lessening our dependency on complicated and opaque rating guidelines, we hope to increase the re-use value of this data set. What you see is what you get!
+
+Demonstrating these subjective labels can be predicted reliably can shine a new light on this research area. Results from this competition will inform the way future intelligent Q&A systems will get built, hopefully contributing to them becoming more human-like.
+
+## Evaluation
+
+Submissions are evaluated on the mean column-wise [Spearman's correlation coefficient](https://en.wikipedia.org/wiki/Spearman%27s_rank_correlation_coefficient). The Spearman's rank correlation is computed for each target column, and the mean of these values is calculated for the submission score.
+
+For each qa_id in the test set, you must predict a probability for each target variable. The predictions should be in the range [0,1]. The file should contain a header and have the following format:
+
+```
+qa_id,question_asker_intent_understanding,...,answer_well_written
+6,0.0,...,0.5
+8,0.5,...,0.1
+18,1.0,...,0.0
+etc.
+```
+
+## Timeline
+
+- February 3, 2020 - Team Merger deadline. This is the last day participants may join or merge teams.
+- February 3, 2020 - Entry deadline. You must accept the competition rules before this date in order to compete.
+- February 10, 2020 - Final submission deadline.
+
+All deadlines are at 11:59 PM UTC on the corresponding day unless otherwise noted. The competition organizers reserve the right to update the contest timeline if they deem it necessary.
+
+## Prizes
+
+- 1st place - $10,000
+- 2nd place - $7,000
+- 3rd place - $5,000
+- 4th place - $2,000
+- 5th place - $1,000
+
+## Notebooks Requirements
+
+Submissions to this competition must be made through Notebooks. Your Notebook will re-run automatically against an unseen test set, and needs to output a file named submission.csv. You can still train a model offline, upload it as a dataset, and use the notebook exclusively to perform inference.
+
+In order for the "Submit to Competition" button to be active after a commit, the following conditions must be met:
+
+  - CPU Notebooks <= 9 hours run-time
+  - GPU Notebooks <= 2 hours run-time
+  - Internet must be turned off
+
+In synchronous Notebooks-only competitions, note that a submission that results in an error--either within the notebook or within the process of scoring--will count against your daily submission limit and will not return the specific error message. This is to limit probing the private test set.
+
+Please note - committing your Notebook will result in a full execution of your code from top-to-bottom.
+
+Please see the [Code Competition FAQ](https://www.kaggle.com/docs/competitions#kernels-only-FAQ) for more information on how to submit.
+
+## Citation
+
+Danicky, Praveen Paritosh, Walter Reade, Addison Howard, Mark McDonald. (2019). Google QUEST Q&A Labeling. Kaggle. https://kaggle.com/competitions/google-quest-challenge
+
+# Dataset Description
+
+The data for this competition includes questions and answers from various StackExchange properties. Your task is to predict target values of 30 labels for each question-answer pair.
+
+The list of 30 target labels are the same as the column names in the `sample_submission.csv` file. Target labels with the prefix `question_` relate to the `question_title` and/or `question_body` features in the data. Target labels with the prefix `answer_` relate to the `answer` feature.
+
+Each row contains a single question and a single answer to that question, along with additional features. The training data contains rows with some duplicated questions (but with different answers). The test data does not contain any duplicated questions.
+
+This is not a binary prediction challenge. Target labels are aggregated from multiple raters, and can have continuous values in the range `[0,1]`. Therefore, predictions must also be in that range.
+
+Since this is a synchronous re-run competition, you only have access to the Public test set. For planning purposes, the re-run test set is no larger than 10,000 rows, and less than 8 Mb uncompressed.
+
+Additional information about the labels and collection method will be provided by the competition sponsor in the forum.
+
+## File descriptions
+
+  - **train.csv** - the training data (target labels are the last 30 columns)
+  - **test.csv** - the test set (you must predict 30 labels for each test set row)
+  - **sample_submission.csv** - a sample submission file in the correct format; column names are the 30 target labels

dslighting/registry/google-quest-challenge/description_obfuscated.md

@@ -0,0 +1,29 @@
+# Task
+
+Given questions and answers from various StackExchange properties, predict target values of 30 labels for each question-answer pair.
+
+# Metric
+
+Mean column-wise Spearman's correlation coefficient. The Spearman's rank correlation is computed for each target column, and the mean of these values is calculated for the submission score.
+
+# Submission Format
+
+For each qa_id in the test set, you must predict a probability for each target variable. The predictions should be in the range [0,1]. The file should contain a header and have the following format:
+
+```
+qa_id,question_asker_intent_understanding,...,answer_well_written
+6,0.0,...,0.5
+8,0.5,...,0.1
+18,1.0,...,0.0
+etc.
+```
+
+# Dataset
+
+The list of 30 target labels are the same as the column names in the `sample_submission.csv` file. Target labels with the prefix `question_` relate to the `question_title` and/or `question_body` features in the data. Target labels with the prefix `answer_` relate to the `answer` feature.
+
+Target labels are aggregated from multiple raters, and can have continuous values in the range `[0,1]`. Therefore, predictions must also be in that range.
+
+- **train.csv** - the training data (target labels are the last 30 columns)
+- **test.csv** - the test set (you must predict 30 labels for each test set row)
+- **sample_submission.csv** - a sample submission file in the correct format; column names are the 30 target labels

dslighting/registry/google-quest-challenge/grade.py

@@ -0,0 +1,45 @@
+import numpy as np
+import pandas as pd
+from scipy.stats import spearmanr
+
+from mlebench.grade_helpers import InvalidSubmissionError
+
+from .classes import CLASSES
+
+
+def grade(submission: pd.DataFrame, answers: pd.DataFrame) -> float:
+    id_col = "qa_id"
+    classes_set = set(CLASSES)
+
+    # submission checks
+    if id_col not in submission.columns:
+        raise InvalidSubmissionError(f"Submission is missing `{id_col}` column")
+    sub_cols_set = set(submission.columns)
+    if not classes_set.issubset(sub_cols_set):
+        raise InvalidSubmissionError(
+            f"Submission is missing the following columns: {classes_set - sub_cols_set}"
+        )
+    if len(submission.columns) != len(CLASSES) + 1:
+        raise InvalidSubmissionError(
+            f"Submission has {len(submission.columns) - 1} columns, expected {len(CLASSES)}"
+        )
+    if len(submission) != len(answers):
+        raise InvalidSubmissionError("Submission and answers have different lengths")
+
+    # answers checks
+    assert id_col in answers.columns, f"Answers is missing `{id_col}` column"
+    ans_cols_set = set(answers.columns)
+    assert classes_set.issubset(
+        ans_cols_set
+    ), f"Answers is missing the following columns: {classes_set - ans_cols_set}"
+    assert (
+        len(answers.columns) == len(CLASSES) + 1
+    ), f"Answers has {len(answers.columns)} columns, expected {len(CLASSES) + 1}"
+
+    # sort by id to ensure correct order
+    submission = submission.sort_values(id_col)
+    answers = answers.sort_values(id_col)
+
+    spearmans = [spearmanr(submission[col], answers[col]).correlation for col in CLASSES]
+    score = np.mean(spearmans)
+    return score