dslighting 1.7.1__py3-none-any.whl → 1.7.6__py3-none-any.whl

mlebench/competitions/osic-pulmonary-fibrosis-progression/grade.py

@@ -0,0 +1,74 @@
+import numpy as np
+import pandas as pd
+from pandas import DataFrame
+
+from mlebench.grade_helpers import InvalidSubmissionError
+
+
+def prepare_for_metric(submission: DataFrame, answers: DataFrame) -> dict:
+
+    assert "Patient_Week" in answers.columns, "Answers DataFrame must have a 'Patient_Week' column."
+    assert "FVC" in answers.columns, "Answers DataFrame must have a 'FVC' column."
+    assert "Patient" in answers.columns, "Answers DataFrame must have a 'Patient' column."
+    if "Patient_Week" not in submission.columns:
+        raise InvalidSubmissionError("Submission DataFrame must have a 'Patient_Week' column.")
+    if "FVC" not in submission.columns:
+        raise InvalidSubmissionError("Submission DataFrame must have a 'FVC' column.")
+    if "Confidence" not in submission.columns:
+        raise InvalidSubmissionError("Submission DataFrame must have a 'Confidence' column.")
+    for pw in submission["Patient_Week"]:
+        if pw not in answers["Patient_Week"].values:
+            raise InvalidSubmissionError(
+                f"Patient_Week {pw} in submission does not exist in answers"
+            )
+    if not pd.api.types.is_numeric_dtype(submission["FVC"]):
+        raise InvalidSubmissionError("FVC column in submission must be numeric.")
+    if not pd.api.types.is_numeric_dtype(submission["Confidence"]):
+        raise InvalidSubmissionError("Confidence column in submission must be numeric.")
+
+    # fillna with 0 for the confidence column
+    submission["Confidence"] = submission["Confidence"].fillna(0)
+
+    # We should only take the 3 latest Patient_Week records for each patient
+    answers = answers.dropna(subset=["FVC"])  # Drop the dummy data
+    answers["Week_Number"] = answers["Patient_Week"].apply(lambda x: int(x.split("_")[-1]))
+    latest_weeks = answers.sort_values("Week_Number").groupby("Patient").tail(3)
+    answers = latest_weeks.drop(columns=["Week_Number"])
+    # Make submission match; we only grade the prediction for the 3 latest weeks
+    submission = submission[submission["Patient_Week"].isin(answers["Patient_Week"])]
+
+    submission = submission.sort_values(by="Patient_Week")
+    answers = answers.sort_values(by="Patient_Week")
+
+    fvc_true = answers.loc[answers["Patient_Week"].isin(submission["Patient_Week"]), "FVC"].values
+    fvc_pred = submission.loc[
+        submission["Patient_Week"].isin(answers["Patient_Week"]), "FVC"
+    ].values
+    confidence = submission.loc[
+        submission["Patient_Week"].isin(answers["Patient_Week"]), "Confidence"
+    ].values
+
+    return {"fvc_true": fvc_true, "fvc_pred": fvc_pred, "confidence": confidence}
+
+
+def laplace_log_likelihood(
+    actual_fvc: np.ndarray, predicted_fvc: np.ndarray, confidence: np.ndarray, return_values=False
+) -> float:
+    """
+    Calculates the modified Laplace Log Likelihood score for osic-pulmonary-fibrosis-progression
+    See https://www.kaggle.com/code/rohanrao/osic-understanding-laplace-log-likelihood for details on this implementation
+    """
+    sd_clipped = np.maximum(confidence, 70)
+    delta = np.minimum(np.abs(actual_fvc - predicted_fvc), 1000)
+    metric = -np.sqrt(2) * delta / sd_clipped - np.log(np.sqrt(2) * sd_clipped)
+
+    if return_values:
+        return metric
+    else:
+        return np.mean(metric)
+
+
+def grade(submission: DataFrame, answers: DataFrame) -> float:
+    fvc_true, fvc_pred, confidence = prepare_for_metric(submission, answers).values()
+
+    return laplace_log_likelihood(fvc_true, fvc_pred, confidence)

mlebench/competitions/osic-pulmonary-fibrosis-progression/prepare.py

@@ -0,0 +1,95 @@
+import shutil
+from multiprocessing import Pool
+from pathlib import Path
+
+import pandas as pd
+from sklearn.model_selection import train_test_split
+from tqdm import tqdm
+
+from mlebench.utils import read_csv
+
+
+def copy_dir(args):
+    src_dir, dst_dir = args
+    shutil.copytree(src=src_dir, dst=dst_dir, dirs_exist_ok=True)
+
+
+def prepare(raw: Path, public: Path, private: Path):
+    # Splitting the train set into train and test with unique Patients
+    old_train = read_csv(raw / "train.csv")
+    grouped_by_patient = list(old_train.groupby("Patient"))
+    train_groups, test_groups = train_test_split(grouped_by_patient, test_size=0.1, random_state=0)
+    new_train = pd.concat([group for _, group in train_groups])
+    new_test = pd.concat([group for _, group in test_groups])
+    assert set(new_train["Patient"]).isdisjoint(
+        set(new_test["Patient"])
+    ), "There are Patients who are in both train and test sets."
+
+    # For the public new_test set we will only keep each patients first FVS measurement. The task is to predict FVS measurements for all possible weeks
+    new_test_public = new_test.sort_values(by="Weeks").groupby("Patient").first().reset_index()
+
+    # Creating the private answers CSV. We need to fill out dummy FVS measurements for all weeks that don't have data so as to match sample_submission.csv
+    # Create a DataFrame with all possible Patient-Week combinations
+    all_weeks = pd.DataFrame(
+        [
+            (patient, week)
+            for patient in new_test["Patient"].unique()
+            for week in range(new_test["Weeks"].min(), new_test["Weeks"].max() + 1)
+        ],
+        columns=["Patient", "Weeks"],
+    )
+    # Merge with the new_test DataFrame to fill in missing weeks with NaN values
+    new_test = all_weeks.merge(new_test, on=["Patient", "Weeks"], how="left")
+    new_test["Patient_Week"] = new_test["Patient"] + "_" + new_test["Weeks"].astype(str)
+    new_test["Confidence"] = 100
+    assert (
+        new_test.groupby("Patient").size().nunique() == 1
+    ), "Not all patients have the same number of rows."
+
+    # Create a sample submission file
+    submission_df = new_test.copy()
+    submission_df = submission_df[["Patient_Week"]]
+    submission_df["FVC"] = 2000  # Dummy predictions
+    submission_df["Confidence"] = 100  # Dummy confidence
+
+    # Write CSVs
+    new_train.to_csv(public / "train.csv", index=False)
+    new_test_public.to_csv(public / "test.csv", index=False)
+    new_test.to_csv(private / "test.csv", index=False)
+    submission_df.to_csv(public / "sample_submission.csv", index=False)
+
+    # Copy over data files
+    (public / "train").mkdir(exist_ok=True)
+    train_args = [
+        (raw / "train" / patient, public / "train" / patient)
+        for patient in new_train["Patient"].unique()
+    ]
+    with Pool() as pool:
+        list(
+            tqdm(pool.imap(copy_dir, train_args), total=len(train_args), desc="Copying train data")
+        )
+
+    (public / "test").mkdir(exist_ok=True)
+    test_args = [
+        (raw / "train" / patient, public / "test" / patient)
+        for patient in new_test["Patient"].unique()
+    ]
+    with Pool() as pool:
+        list(tqdm(pool.imap(copy_dir, test_args), total=len(test_args), desc="Copying test data"))
+
+    # Final checks
+    assert new_train.shape[1] == 7, f"Expected 7 columns in new_train, but got {new_train.shape[1]}"
+    assert new_test.shape[1] == 9, f"Expected 9 columns in new_test, but got {new_test.shape[1]}"
+    assert (
+        new_test_public.shape[1] == 7
+    ), f"Expected 7 columns in new_test_public, but got {new_test_public.shape[1]}"
+    assert (
+        submission_df.shape[1] == 3
+    ), f"Expected 3 columns in submission_df, but got {submission_df.shape[1]}"
+
+    public_train_dirs = set((public / "train").iterdir())
+    public_test_dirs = set((public / "test").iterdir())
+    common_dirs = public_train_dirs.intersection(public_test_dirs)
+    assert (
+        not common_dirs
+    ), f"There are directories with the same name in public train and test: {common_dirs}"

mlebench/competitions/osic-pulmonary-fibrosis-progression/prepare_val.py

@@ -0,0 +1,167 @@
+import shutil
+from multiprocessing import Pool
+from pathlib import Path
+from typing import List, Tuple
+
+import pandas as pd
+from sklearn.model_selection import train_test_split
+from tqdm import tqdm
+
+from mlebench.utils import read_csv
+
+
+def copy_dir(args):
+    src_dir, dst_dir = args
+    shutil.copytree(src=src_dir, dst=dst_dir, dirs_exist_ok=True)
+
+
+def _create_dataset_split(
+    train_groups: List[Tuple[str, pd.DataFrame]],
+    test_groups: List[Tuple[str, pd.DataFrame]],
+    raw_path: Path,
+    public_path: Path,
+    private_path: Path,
+):
+    """
+    Helper function to process a single train/test split of patient groups.
+
+    This function takes lists of train and test patient groups, creates the necessary
+    DataFrames, saves them to the specified public and private directories, and copies
+    the associated patient data folders. This logic is used for both the original
+    train/test split and the new train/validation split.
+    """
+    # Ensure output directories exist
+    public_path.mkdir(exist_ok=True)
+    private_path.mkdir(exist_ok=True)
+
+    # Recreate DataFrames from the split groups
+    new_train = pd.concat([group for _, group in train_groups])
+    new_test = pd.concat([group for _, group in test_groups])
+    assert set(new_train["Patient"]).isdisjoint(
+        set(new_test["Patient"])
+    ), "There are Patients who are in both train and test sets."
+
+    # For the public new_test set we will only keep each patients first FVS measurement. The task is to predict FVS measurements for all possible weeks
+    new_test_public = new_test.sort_values(by="Weeks").groupby("Patient").first().reset_index()
+
+    # Creating the private answers CSV. We need to fill out dummy FVS measurements for all weeks that don't have data so as to match sample_submission.csv
+    # Create a DataFrame with all possible Patient-Week combinations
+    all_weeks = pd.DataFrame(
+        [
+            (patient, week)
+            for patient in new_test["Patient"].unique()
+            for week in range(new_test["Weeks"].min(), new_test["Weeks"].max() + 1)
+        ],
+        columns=["Patient", "Weeks"],
+    )
+    # Merge with the new_test DataFrame to fill in missing weeks with NaN values
+    new_test_private = all_weeks.merge(new_test, on=["Patient", "Weeks"], how="left")
+    new_test_private["Patient_Week"] = (
+        new_test_private["Patient"] + "_" + new_test_private["Weeks"].astype(str)
+    )
+    new_test_private["Confidence"] = 100
+    assert (
+        new_test_private.groupby("Patient").size().nunique() == 1
+    ), "Not all patients have the same number of rows."
+
+    # Create a sample submission file
+    submission_df = new_test_private.copy()
+    submission_df = submission_df[["Patient_Week"]]
+    submission_df["FVC"] = 2000  # Dummy predictions
+    submission_df["Confidence"] = 100  # Dummy confidence
+
+    # Write CSVs
+    new_train.to_csv(public_path / "train.csv", index=False)
+    new_test_public.to_csv(public_path / "test.csv", index=False)
+    new_test_private.to_csv(private_path / "test.csv", index=False)
+    submission_df.to_csv(public_path / "sample_submission.csv", index=False)
+
+    # Copy over data files
+    (public_path / "train").mkdir(exist_ok=True)
+    train_args = [
+        (raw_path / "train" / patient, public_path / "train" / patient)
+        for patient in new_train["Patient"].unique()
+    ]
+    with Pool() as pool:
+        list(
+            tqdm(
+                pool.imap(copy_dir, train_args),
+                total=len(train_args),
+                desc=f"Copying train data to {public_path.name}",
+            )
+        )
+
+    (public_path / "test").mkdir(exist_ok=True)
+    test_args = [
+        (raw_path / "train" / patient, public_path / "test" / patient)
+        for patient in new_test["Patient"].unique()
+    ]
+    with Pool() as pool:
+        list(
+            tqdm(
+                pool.imap(copy_dir, test_args),
+                total=len(test_args),
+                desc=f"Copying test data to {public_path.name}",
+            )
+        )
+
+    # Final checks
+    assert new_train.shape[1] == 7, f"Expected 7 columns in new_train, but got {new_train.shape[1]}"
+    assert (
+        new_test_private.shape[1] == 9
+    ), f"Expected 9 columns in new_test, but got {new_test_private.shape[1]}"
+    assert (
+        new_test_public.shape[1] == 7
+    ), f"Expected 7 columns in new_test_public, but got {new_test_public.shape[1]}"
+    assert (
+        submission_df.shape[1] == 3
+    ), f"Expected 3 columns in submission_df, but got {submission_df.shape[1]}"
+
+    public_train_dirs = set((public_path / "train").iterdir())
+    public_test_dirs = set((public_path / "test").iterdir())
+    common_dirs = public_train_dirs.intersection(public_test_dirs)
+    assert (
+        not common_dirs
+    ), f"There are directories with the same name in public train and test: {common_dirs}"
+
+
+def prepare(raw: Path, public: Path, private: Path):
+    # Read raw data and group by patient to ensure patient-level splits
+    old_train = read_csv(raw / "train.csv")
+    grouped_by_patient = list(old_train.groupby("Patient"))
+
+    # ---- 1. Create the original train/test split ----
+    # This split creates the main competition data in `public` and `private`
+    train_groups, test_groups = train_test_split(
+        grouped_by_patient, test_size=0.1, random_state=0
+    )
+    _create_dataset_split(
+        train_groups=train_groups,
+        test_groups=test_groups,
+        raw_path=raw,
+        public_path=public,
+        private_path=private,
+    )
+
+    # ---- 2. Create the new train/validation split ----
+    # This second split uses the `train_groups` from the first split to create
+    # a smaller training set and a validation set. The outputs are saved in
+    # parallel directories (`public_val`, `private_val`) to avoid altering
+    # the original competition files.
+
+    # Define new paths for the validation split
+    public_val = public.parent / "public_val"
+    private_val = private.parent / "private_val"
+
+    # Split the training data again to create a validation set.
+    # We use the *same logic and random_state* as the original split.
+    train_val_groups, test_val_groups = train_test_split(
+        train_groups, test_size=0.1, random_state=0
+    )
+    _create_dataset_split(
+        train_groups=train_val_groups,
+        test_groups=test_val_groups,
+        raw_path=raw,
+        public_path=public_val,
+        private_path=private_val,
+    )

mlebench/competitions/paddy-disease-classification/grade.py

@@ -0,0 +1,35 @@
+from pandas import DataFrame
+from sklearn.metrics import accuracy_score
+
+from mlebench.grade_helpers import InvalidSubmissionError
+
+
+def grade(submission: DataFrame, answers: DataFrame) -> float:
+    expected_columns = set(["image_id", "label"])
+
+    assert set(answers.columns).issuperset(
+        expected_columns
+    ), f"Expected columns {expected_columns}, but got {set(answers.columns)}"
+
+    assert not answers.isnull().values.any(), "Answers contain NaN values"
+
+    if not set(submission.columns).issuperset(expected_columns):
+        raise InvalidSubmissionError(
+            f"Expected columns {expected_columns}, but got {set(submission.columns)}"
+        )
+
+    if len(submission) != len(answers):
+        raise InvalidSubmissionError(f"Expected {len(answers)} rows, but got {len(submission)}")
+
+    sorted_submission = submission.sort_values(by="image_id").reset_index(drop=True)
+    sorted_answers = answers.sort_values(by="image_id").reset_index(drop=True)
+
+    if (sorted_submission["image_id"].values != sorted_answers["image_id"].values).any():
+        raise InvalidSubmissionError("Invalid image IDs in the submission!")
+
+    y_pred = sorted_submission["label"].fillna("").tolist()
+    y_true = sorted_answers["label"]
+
+    score = accuracy_score(y_true=y_true, y_pred=y_pred)
+
+    return score

mlebench/competitions/paddy-disease-classification/prepare.py

@@ -0,0 +1,69 @@
+import shutil
+from pathlib import Path
+
+from sklearn.model_selection import train_test_split
+
+from mlebench.utils import read_csv
+
+
+def copy(src: Path, dst: Path):
+    """A wrapper for `shutil.copy` which creates destination directories when they don't exist."""
+
+    assert src.exists(), f"{src} does not exist"
+    assert not dst.exists(), f"{dst} already exists"
+    dst.parent.mkdir(parents=True, exist_ok=True)
+    shutil.copy(src, dst)
+
+
+def prepare(raw: Path, public: Path, private: Path):
+    old_train = read_csv(raw / "train.csv")
+    old_sample_submission = read_csv(raw / "sample_submission.csv")
+
+    # The original dataset has 10,407 train images and 3,469 test images.
+    # This implies a 75%/25% train/test split.
+    new_train, new_test = train_test_split(old_train, train_size=0.75, random_state=0)
+
+    new_sample_submission = new_test[["image_id", "label"]].copy()
+    new_sample_submission["label"] = ""
+
+    new_train.to_csv(public / "train.csv", index=False)
+    new_sample_submission.to_csv(public / "sample_submission.csv", index=False)
+    new_test.to_csv(private / "test.csv", index=False)
+
+    for row in new_train.itertuples():
+        copy(
+            raw / "train_images" / row.label / row.image_id,
+            public / "train_images" / row.label / row.image_id,
+        )
+
+    for row in new_test.itertuples():
+        copy(
+            raw / "train_images" / row.label / row.image_id,
+            public / "test_images" / row.image_id,
+        )
+
+    # Sanity checks
+    assert len(new_train) + len(new_test) == len(
+        old_train
+    ), f"Expected the combined size of the new train and test sets to be the same size as the original train set, but it wasn't! Got {len(new_train) + len(new_test)} != {len(old_train)}."
+
+    new_train_image_ids = set(new_train.image_id)
+    new_test_image_ids = set(new_test.image_id)
+
+    assert new_train_image_ids.isdisjoint(new_test_image_ids), "Train and test sets overlap!"
+
+    assert set(new_train.columns) == set(
+        old_train.columns
+    ), f"Expected the new train set to have the same columns as the original train set, but it didn't! Got {set(new_train.columns)} != {set(old_train.columns)}."
+
+    assert set(new_sample_submission.columns) == set(
+        old_sample_submission.columns
+    ), f"Expected the new sample submission to have the same columns as the original sample submission, but it didn't! Got {set(new_sample_submission.columns)} != {set(old_sample_submission.columns)}."
+
+    assert len(list(public.glob("train_images/*/*.jpg"))) == len(
+        new_train
+    ), f"Expected the number of images in the `public / train_images` directory to match the number of rows in the `public / train.csv` file, but it didn't! Got {len(list(public.glob('train_images/*/*.jpg')))} != {len(new_train)}."
+
+    assert len(list(public.glob("test_images/*.jpg"))) == len(
+        new_test
+    ), f"Expected the number of images in the `public / test_images` directory to match the number of rows in the `private / test.csv` file, but it didn't! Got {len(list(public.glob('test_images/*.jpg')))} != {len(new_test)}."

mlebench/competitions/paddy-disease-classification/prepare_val.py

@@ -0,0 +1,122 @@
+import shutil
+from pathlib import Path
+
+import pandas as pd
+from sklearn.model_selection import train_test_split
+
+from mlebench.utils import read_csv
+
+
+def copy(src: Path, dst: Path):
+    """A wrapper for `shutil.copy` which creates destination directories when they don't exist."""
+
+    assert src.exists(), f"{src} does not exist"
+    # Allow overwriting for simplicity in rerunning the script
+    if dst.exists():
+        if dst.is_dir():
+            shutil.rmtree(dst)
+        else:
+            dst.unlink()
+    dst.parent.mkdir(parents=True, exist_ok=True)
+    shutil.copy(src, dst)
+
+
+def _process_and_save_split(
+    train_df: pd.DataFrame,
+    test_df: pd.DataFrame,
+    public_path: Path,
+    private_path: Path,
+    raw_path: Path,
+    old_sample_submission: pd.DataFrame,
+):
+    """
+    Helper function to process a single train/test split and save the results.
+    This encapsulates the logic for creating CSVs, copying images, and running checks.
+    """
+    sample_submission = test_df[["image_id", "label"]].copy()
+    sample_submission["label"] = ""
+
+    # Ensure output directories exist
+    public_path.mkdir(exist_ok=True)
+    private_path.mkdir(exist_ok=True)
+
+    train_df.to_csv(public_path / "train.csv", index=False)
+    sample_submission.to_csv(public_path / "sample_submission.csv", index=False)
+    test_df.to_csv(private_path / "test.csv", index=False)
+
+    for row in train_df.itertuples():
+        copy(
+            raw_path / "train_images" / row.label / row.image_id,
+            public_path / "train_images" / row.label / row.image_id,
+        )
+
+    for row in test_df.itertuples():
+        copy(
+            raw_path / "train_images" / row.label / row.image_id,
+            public_path / "test_images" / row.image_id,
+        )
+
+    # Sanity checks (generalized for any split)
+    train_image_ids = set(train_df.image_id)
+    test_image_ids = set(test_df.image_id)
+
+    assert train_image_ids.isdisjoint(test_image_ids), f"Train and test sets overlap in {public_path}!"
+
+    assert set(train_df.columns) == set(
+        test_df.columns
+    ), f"Expected the new train and test sets to have the same columns, but they didn't! Got {set(train_df.columns)} != {set(test_df.columns)} in {public_path}."
+
+    assert set(sample_submission.columns) == set(
+        old_sample_submission.columns
+    ), f"Expected the new sample submission to have the same columns as the original sample submission, but it didn't! Got {set(sample_submission.columns)} != {set(old_sample_submission.columns)} in {public_path}."
+
+    assert len(list(public_path.glob("train_images/*/*.jpg"))) == len(
+        train_df
+    ), f"Expected the number of images in the `{public_path / 'train_images'}` directory to match the number of rows in the `{public_path / 'train.csv'}` file, but it didn't! Got {len(list(public_path.glob('train_images/*/*.jpg')))} != {len(train_df)}."
+
+    assert len(list(public_path.glob("test_images/*.jpg"))) == len(
+        test_df
+    ), f"Expected the number of images in the `{public_path / 'test_images'}` directory to match the number of rows in the `{private_path / 'test.csv'}` file, but it didn't! Got {len(list(public_path.glob('test_images/*.jpg')))} != {len(test_df)}."
+
+
+def prepare(raw: Path, public: Path, private: Path):
+    old_train = read_csv(raw / "train.csv")
+    old_sample_submission = read_csv(raw / "sample_submission.csv")
+
+    # The original dataset has 10,407 train images and 3,469 test images.
+    # This implies a 75%/25% train/test split.
+    new_train, new_test = train_test_split(old_train, train_size=0.75, random_state=0)
+
+    # Process and save the original split to `public` and `private` directories
+    _process_and_save_split(
+        train_df=new_train,
+        test_df=new_test,
+        public_path=public,
+        private_path=private,
+        raw_path=raw,
+        old_sample_submission=old_sample_submission,
+    )
+
+    # Sanity check specific to the first split
+    assert len(new_train) + len(new_test) == len(
+        old_train
+    ), f"Expected the combined size of the new train and test sets to be the same size as the original train set, but it wasn't! Got {len(new_train) + len(new_test)} != {len(old_train)}."
+
+    # --- New Validation Set Creation ---
+    # Define paths for the new validation split
+    public_val = public.parent / "public_val"
+    private_val = private.parent / "private_val"
+
+    # Perform a second split on the `new_train` set created above.
+    # Use the exact same splitting parameters to replicate the logic.
+    train_val, test_val = train_test_split(new_train, train_size=0.75, random_state=0)
+
+    # Process and save the new validation split to `public_val` and `private_val`
+    _process_and_save_split(
+        train_df=train_val,
+        test_df=test_val,
+        public_path=public_val,
+        private_path=private_val,
+        raw_path=raw,
+        old_sample_submission=old_sample_submission,
+    )

mlebench/competitions/petfinder-pawpularity-score/grade.py

@@ -0,0 +1,41 @@
+import pandas as pd
+from sklearn.metrics import root_mean_squared_error
+
+from mlebench.grade_helpers import InvalidSubmissionError
+
+
+def prepare_for_metric(submission: pd.DataFrame, answers: pd.DataFrame) -> dict:
+    id_col = "Id"
+    target_col = "Pawpularity"
+
+    assert id_col in answers.columns, f"Missing {id_col} column in answers"
+    assert target_col in answers.columns, f"Missing {target_col} column in answers"
+    assert (
+        answers[target_col].between(1, 100).all()
+    ), f"Pawpularity in answers should be between 1 and 100"
+
+    if id_col not in submission.columns:
+        raise InvalidSubmissionError(f"Missing {id_col} column in submission")
+
+    if target_col not in submission.columns:
+        raise InvalidSubmissionError(f"Missing {target_col} column in submission")
+
+    if not submission[target_col].between(1, 100).all():
+        raise InvalidSubmissionError(f"Pawpularity in submission should be between 1 and 100")
+
+    submission = submission.sort_values(id_col)
+    answers = answers.sort_values(id_col)
+
+    if (submission[id_col].values != answers[id_col].values).any():
+        raise InvalidSubmissionError(f"IDs in submission do not match IDs in answers")
+
+    return {
+        "y_true": answers[target_col].to_numpy(),
+        "y_pred": submission[target_col].to_numpy(),
+    }
+
+
+def grade(submission: pd.DataFrame, answers: pd.DataFrame) -> float:
+    rmse_input = prepare_for_metric(submission, answers)
+    score = root_mean_squared_error(**rmse_input)
+    return score