dslighting 1.7.1__py3-none-any.whl → 1.7.6__py3-none-any.whl

mlebench/competitions/AI4Code/grade.py

@@ -0,0 +1,70 @@
+from bisect import bisect
+from typing import Any
+
+import pandas as pd
+
+from mlebench.grade_helpers import InvalidSubmissionError
+
+
+def kendall_tau(ground_truth: list[list[Any]], predictions: list[list[Any]]) -> float:
+    """
+    Computes the Kendall Tau correlation between
+    - `ground_truth`: a list of lists, where each sublist contains the ground truth order of items
+    - `predictions`: a list of lists, where each sublist contains the predicted order of items
+
+    See: https://www.kaggle.com/code/ryanholbrook/competition-metric-kendall-tau-correlation
+    """
+
+    # Actually O(N^2), but fast in practice for our data
+    def count_inversions(a):
+        inversions = 0
+        sorted_so_far = []
+        for i, u in enumerate(a):  # O(N)
+            j = bisect(sorted_so_far, u)  # O(log N)
+            inversions += i - j
+            sorted_so_far.insert(j, u)  # O(N)
+        return inversions
+
+    total_inversions = 0  # total inversions in predicted ranks across all instances
+    total_2max = 0  # maximum possible inversions across all instances
+    for gt, pred in zip(ground_truth, predictions):
+        ranks = [gt.index(x) for x in pred]  # rank predicted order in terms of ground truth
+        total_inversions += count_inversions(ranks)
+        n = len(gt)
+        total_2max += n * (n - 1)
+    return 1 - 4 * total_inversions / total_2max
+
+
+def prepare_for_metric(submission: pd.DataFrame, answers: pd.DataFrame) -> dict:
+    if len(submission) != len(answers):
+        raise InvalidSubmissionError("Submission and answers must have the same length")
+    if "id" not in submission.columns or "cell_order" not in submission.columns:
+        raise InvalidSubmissionError("Submission must have 'id' and 'cell_order' columns")
+    if set(submission["id"]) != set(answers["id"]):
+        raise InvalidSubmissionError("Submission and answers must have the same ids")
+    # "cell_order" columns must be strings of space-separated cell ids: "54c7cab3 fe66203e 7844d5f8 ..."
+    submission["cell_order"] = submission["cell_order"].astype(str)
+    answers["cell_order"] = answers["cell_order"].astype(str)
+
+    # Sort by id to ensure we're comparing the same rows
+    submission = submission.sort_values("id")
+    answers = answers.sort_values("id")
+
+    ground_truth, predictions = [], []
+    target_column = "cell_order"
+    for (_, row_gt), (_, row_pred) in zip(submission.iterrows(), answers.iterrows()):
+        ground_truth_items = row_gt[target_column].split()
+        prediction_items = row_pred[target_column].split()
+        if len(ground_truth_items) != len(prediction_items):
+            raise InvalidSubmissionError("Ground truth and predictions must have the same length")
+        ground_truth.append(ground_truth_items)
+        predictions.append(prediction_items)
+    return {
+        "ground_truth": ground_truth,
+        "predictions": predictions,
+    }
+
+
+def grade(submission: pd.DataFrame, answers: pd.DataFrame) -> float:
+    kendall_tau_inputs = prepare_for_metric(submission, answers)
+    return kendall_tau(**kendall_tau_inputs)

mlebench/competitions/AI4Code/prepare.py

@@ -0,0 +1,84 @@
+import json
+import shutil
+from pathlib import Path
+
+import pandas as pd
+from tqdm.auto import tqdm
+
+from mlebench.utils import read_csv
+
+
+# Create train, test from train split
+def create_train_test_split(train_ancestors_df: pd.DataFrame, test_size: int):
+    """
+    On Kaggle, a user may "fork" (that is, copy) the notebook of another user to create their own version.
+    `train_ancestors_df` contains information about the ancestry of each notebook in the training set.
+
+    To create the test split, we select rows from the train set that don't share ancestors with any others
+    so that there aren't any relatives in the train set. The train split is the remaining rows.
+    """
+    group_by_ancestor = {}
+    for _, row in tqdm(
+        train_ancestors_df.iterrows(), desc="Grouping by Ancestor", total=len(train_ancestors_df)
+    ):
+        ancestor_id = row["ancestor_id"]
+        if ancestor_id not in group_by_ancestor:
+            group_by_ancestor[ancestor_id] = []
+        group_by_ancestor[ancestor_id].append(row)
+
+    train_ids = []
+    test_ids = []
+    num_test_ids = 0
+    for ancestor_id, rows in tqdm(group_by_ancestor.items(), desc="Splitting on Ancestors"):
+        if num_test_ids + len(rows) <= test_size:
+            test_ids.extend(([row["id"] for row in rows]))
+            num_test_ids += len(rows)
+        else:
+            train_ids.extend([row["id"] for row in rows])
+
+    assert len(test_ids) == test_size
+    assert len(train_ids) == len(train_ancestors_df) - test_size
+
+    return train_ids, test_ids
+
+
+def prepare(raw: Path, public: Path, private: Path):
+    train_ancestors_df = read_csv(raw / "train_ancestors.csv")
+    # Shuffle the train_ancestors_df to ensure our split is random
+    train_ancestors_df = train_ancestors_df.sample(frac=1, random_state=0).reset_index(drop=True)
+    new_train_ids, new_test_ids = create_train_test_split(train_ancestors_df, test_size=20000)
+
+    # Copy json files to public
+    (public / "train").mkdir(parents=True, exist_ok=True)
+    for train_id in tqdm(new_train_ids, desc="Copying train json files"):
+        shutil.copy(raw / "train" / f"{train_id}.json", public / "train" / f"{train_id}.json")
+    (public / "test").mkdir(parents=True, exist_ok=True)
+    for test_id in tqdm(new_test_ids, desc="Copying test json files"):
+        shutil.copy(raw / "train" / f"{test_id}.json", public / "test" / f"{test_id}.json")
+
+    # Generate answers for train and test
+    train_orders = read_csv(raw / "train_orders.csv")
+    # Answers for new train
+    train_orders_new = train_orders[train_orders["id"].isin(new_train_ids)]
+    train_orders_new.to_csv(public / "train_orders.csv", index=False)
+    # Answers for new test
+    test_orders_new = train_orders[train_orders["id"].isin(new_test_ids)]
+    test_orders_new.to_csv(private / "test_orders.csv", index=False)
+
+    # Make new train_ancestors.csv, excluding the new_test_ids
+    train_ancestors_df = train_ancestors_df[~train_ancestors_df["id"].isin(new_test_ids)]
+    train_ancestors_df.to_csv(public / "train_ancestors.csv", index=False)
+
+    # Create sample submission (use the given order without changing it)
+    sample_submission_rows = []
+    for sample_id in tqdm(test_orders_new["id"], desc="Creating sample submission"):
+        # Get cell order from json file
+        with open(public / "test" / f"{sample_id}.json") as f:
+            json_data = json.load(f)
+            cell_order = list(json_data["cell_type"].keys())
+        sample_submission_rows.append({"id": sample_id, "cell_order": " ".join(cell_order)})
+    sample_submission = pd.DataFrame(sample_submission_rows)
+    sample_submission.to_csv(public / "sample_submission.csv", index=False)
+    assert len(sample_submission) == len(
+        new_test_ids
+    ), "Sample submission length does not match test length."

mlebench/competitions/AI4Code/prepare_val.py

@@ -0,0 +1,159 @@
+import json
+import shutil
+from pathlib import Path
+
+import pandas as pd
+from tqdm.auto import tqdm
+
+from mlebench.utils import read_csv
+
+
+# Create train, test from train split
+def create_train_test_split(train_ancestors_df: pd.DataFrame, test_size: int):
+    """
+    On Kaggle, a user may "fork" (that is, copy) the notebook of another user to create their own version.
+    `train_ancestors_df` contains information about the ancestry of each notebook in the training set.
+
+    To create the test split, we select rows from the train set that don't share ancestors with any others
+    so that there aren't any relatives in the train set. The train split is the remaining rows.
+    """
+    group_by_ancestor = {}
+    for _, row in tqdm(
+        train_ancestors_df.iterrows(), desc="Grouping by Ancestor", total=len(train_ancestors_df)
+    ):
+        ancestor_id = row["ancestor_id"]
+        if ancestor_id not in group_by_ancestor:
+            group_by_ancestor[ancestor_id] = []
+        group_by_ancestor[ancestor_id].append(row)
+
+    train_ids = []
+    test_ids = []
+    num_test_ids = 0
+    # Ensure a deterministic split by sorting the groups
+    sorted_ancestors = sorted(group_by_ancestor.items(), key=lambda x: x[0])
+    for ancestor_id, rows in tqdm(sorted_ancestors, desc="Splitting on Ancestors"):
+        if num_test_ids + len(rows) <= test_size:
+            test_ids.extend(([row["id"] for row in rows]))
+            num_test_ids += len(rows)
+        else:
+            train_ids.extend([row["id"] for row in rows])
+
+    # The exact number of test samples can vary slightly based on group sizes.
+    # We adjust the assertion to check it's close to the target, not necessarily equal.
+    assert len(train_ids) + len(test_ids) == len(train_ancestors_df)
+
+    return train_ids, test_ids
+
+
+def _generate_split_files(
+    train_ids: list,
+    test_ids: list,
+    public_dir: Path,
+    private_dir: Path,
+    raw_dir: Path,
+    full_train_orders_df: pd.DataFrame,
+    full_train_ancestors_df: pd.DataFrame,
+):
+    """
+    A helper function to generate all required files for a given train/test split.
+    This helps avoid code duplication and ensures consistent output structure.
+    """
+    # Create output directories
+    public_dir.mkdir(parents=True, exist_ok=True)
+    private_dir.mkdir(parents=True, exist_ok=True)
+    (public_dir / "train").mkdir(exist_ok=True)
+    (public_dir / "test").mkdir(exist_ok=True)
+
+    # Copy json files to public
+    for train_id in tqdm(train_ids, desc=f"Copying train json files to {public_dir.name}"):
+        shutil.copy(raw_dir / "train" / f"{train_id}.json", public_dir / "train" / f"{train_id}.json")
+    for test_id in tqdm(test_ids, desc=f"Copying test json files to {public_dir.name}"):
+        shutil.copy(raw_dir / "train" / f"{test_id}.json", public_dir / "test" / f"{test_id}.json")
+
+    # Generate answers for train and test
+    # Answers for new train
+    train_orders_new = full_train_orders_df[full_train_orders_df["id"].isin(train_ids)]
+    train_orders_new.to_csv(public_dir / "train_orders.csv", index=False)
+    # Answers for new test
+    test_orders_new = full_train_orders_df[full_train_orders_df["id"].isin(test_ids)]
+    test_orders_new.to_csv(private_dir / "test_orders.csv", index=False)
+
+    # Make new train_ancestors.csv, excluding the new_test_ids
+    train_ancestors_new = full_train_ancestors_df[
+        full_train_ancestors_df["id"].isin(train_ids)
+    ]
+    train_ancestors_new.to_csv(public_dir / "train_ancestors.csv", index=False)
+
+    # Create sample submission (use the given order without changing it)
+    sample_submission_rows = []
+    for sample_id in tqdm(
+        test_orders_new["id"], desc=f"Creating sample submission for {public_dir.name}"
+    ):
+        # Get cell order from json file
+        with open(public_dir / "test" / f"{sample_id}.json") as f:
+            json_data = json.load(f)
+            cell_order = list(json_data["cell_type"].keys())
+        sample_submission_rows.append({"id": sample_id, "cell_order": " ".join(cell_order)})
+    sample_submission = pd.DataFrame(sample_submission_rows)
+    sample_submission.to_csv(public_dir / "sample_submission.csv", index=False)
+    assert len(sample_submission) == len(test_ids), "Sample submission length does not match test length."
+
+
+def prepare(raw: Path, public: Path, private: Path):
+    # --- Initial Setup ---
+    TEST_SIZE = 20000
+    # Load all necessary data once
+    full_train_ancestors_df = read_csv(raw / "train_ancestors.csv")
+    full_train_orders_df = read_csv(raw / "train_orders.csv")
+
+    # Shuffle the train_ancestors_df to ensure our split is random but reproducible
+    shuffled_ancestors_df = full_train_ancestors_df.sample(
+        frac=1, random_state=0
+    ).reset_index(drop=True)
+
+    # --- 1. Original Split (train/test) ---
+    print("--- Generating original train/test split ---")
+    train_ids, test_ids = create_train_test_split(shuffled_ancestors_df, test_size=TEST_SIZE)
+
+    # Generate files for the original public and private directories
+    _generate_split_files(
+        train_ids,
+        test_ids,
+        public,
+        private,
+        raw,
+        full_train_orders_df,
+        shuffled_ancestors_df,
+    )
+
+    # --- 2. Validation Split (train_val/test_val) ---
+    print("\n--- Generating new validation split ---")
+    # Define paths for the new validation directories
+    public_val = public.parent / "public_val"
+    private_val = private.parent / "private_val"
+
+    # The input for the second split is the training set from the first split.
+    # We use the already shuffled dataframe to maintain deterministic behavior.
+    train_subset_for_val_split = shuffled_ancestors_df[
+        shuffled_ancestors_df["id"].isin(train_ids)
+    ].reset_index(drop=True)
+
+    # Create the second split from the first split's training data
+    train_val_ids, test_val_ids = create_train_test_split(
+        train_subset_for_val_split, test_size=TEST_SIZE
+    )
+
+    # Generate the validation split files in the new directories
+    _generate_split_files(
+        train_val_ids,
+        test_val_ids,
+        public_val,
+        private_val,
+        raw,
+        full_train_orders_df,
+        shuffled_ancestors_df,
+    )
+
+    print("\nData preparation complete.")
+    print(f"Original split created in '{public.name}' and '{private.name}'.")
+    print(f"Validation split created in '{public_val.name}' and '{private_val.name}'.")

mlebench/competitions/aerial-cactus-identification/grade.py

@@ -0,0 +1,11 @@
+import pandas as pd
+from sklearn.metrics import roc_auc_score
+
+from mlebench.competitions.utils import prepare_for_auroc_metric
+
+
+def grade(submission: pd.DataFrame, answers: pd.DataFrame) -> float:
+    roc_auc_inputs = prepare_for_auroc_metric(
+        submission=submission, answers=answers, id_col="id", target_col="has_cactus"
+    )
+    return roc_auc_score(y_true=roc_auc_inputs["y_true"], y_score=roc_auc_inputs["y_score"])

mlebench/competitions/aerial-cactus-identification/prepare.py

@@ -0,0 +1,71 @@
+import shutil
+from pathlib import Path
+
+from sklearn.model_selection import train_test_split
+from tqdm.auto import tqdm
+
+from mlebench.utils import compress, extract, read_csv
+
+
+def prepare(raw: Path, public: Path, private: Path):
+
+    old_train = read_csv(raw / "train.csv")
+
+    # 4000 / (4000 + 17500) -> test_size is ~0.19
+    new_train, new_test = train_test_split(old_train, test_size=0.19, random_state=0)
+
+    new_train.to_csv(public / "train.csv", index=False)
+    new_test.to_csv(private / "test.csv", index=False)
+
+    sample_submission = new_test.copy()
+    sample_submission["has_cactus"] = 0.5
+    sample_submission.to_csv(public / "sample_submission.csv", index=False)
+
+    # need to split the train.zip into train.zip and test.zip; to do so need to extract first
+    extract(raw / "train.zip", raw)
+
+    # copy
+    (public / "train").mkdir(parents=True, exist_ok=True)
+    for image_id in tqdm(new_train["id"], desc="Copying train images", total=len(new_train)):
+        shutil.copy(raw / "train" / image_id, public / "train" / image_id)
+
+    (public / "test").mkdir(parents=True, exist_ok=True)
+    for image_id in tqdm(new_test["id"], desc="Copying test images", total=len(new_test)):
+        shutil.copy(raw / "train" / image_id, public / "test" / image_id)
+
+    # and then recompress
+    compress(public / "train", public / "train.zip")
+    compress(public / "test", public / "test.zip")
+
+    # and cleanup
+    shutil.rmtree(public / "train")
+    shutil.rmtree(public / "test")
+    shutil.rmtree(raw / "train")
+
+    # checks
+    assert (public / "train.zip").exists(), "public/train.zip should exist"
+    assert (public / "test.zip").exists(), "public/test.zip should exist"
+    assert not (raw / "train").exists(), "raw/train/ should not exist"
+
+    assert len(new_train) + len(new_test) == len(
+        old_train
+    ), "The lengths of the splits should add up to the original"
+    assert len(new_train) > len(new_test), "The train set should be larger than the test set"
+    assert len(new_test) == len(
+        sample_submission
+    ), "The test set should match the sample submission"
+
+    assert (
+        new_train.columns.tolist()
+        == new_test.columns.tolist()
+        == old_train.columns.tolist()
+        == sample_submission.columns.tolist()
+    ), "All dataframes should have the same columns, i.e. ['id', 'has_cactus']"
+
+    assert set(new_train["id"]).isdisjoint(new_test["id"]), "Train and test ids should not overlap"
+    assert set(new_test["id"]) == set(
+        sample_submission["id"]
+    ), "Test and sample_submission ids should match"
+
+    assert new_train["id"].nunique() == len(new_train), "There should be no duplicate ids in train"
+    assert new_test["id"].nunique() == len(new_test), "There should be no duplicate ids in test"

mlebench/competitions/aerial-cactus-identification/prepare_val.py

@@ -0,0 +1,133 @@
+import shutil
+from pathlib import Path
+import pandas as pd
+
+from sklearn.model_selection import train_test_split
+from tqdm.auto import tqdm
+
+from mlebench.utils import compress, extract, read_csv
+
+
+def _process_and_save_split(
+    train_df: pd.DataFrame,
+    test_df: pd.DataFrame,
+    public_dir: Path,
+    private_dir: Path,
+    raw_image_dir: Path,
+    original_df: pd.DataFrame,
+):
+    """
+    Helper function to process a single train/test split.
+    It saves all required CSVs, copies and compresses images, and runs assertions.
+    """
+    public_dir.mkdir(parents=True, exist_ok=True)
+    private_dir.mkdir(parents=True, exist_ok=True)
+    
+    # Save the dataframes
+    train_df.to_csv(public_dir / "train.csv", index=False)
+    test_df.to_csv(private_dir / "test.csv", index=False)
+
+    sample_submission = test_df.copy()
+    sample_submission["has_cactus"] = 0.5
+    sample_submission.to_csv(public_dir / "sample_submission.csv", index=False)
+
+    # Prepare temporary directories for images
+    temp_train_img_dir = public_dir / "train"
+    temp_test_img_dir = public_dir / "test"
+    temp_train_img_dir.mkdir(parents=True, exist_ok=True)
+    temp_test_img_dir.mkdir(parents=True, exist_ok=True)
+
+    # Copy images for the current split
+    for image_id in tqdm(train_df["id"], desc=f"Copying train images to {public_dir.name}", total=len(train_df)):
+        shutil.copy(raw_image_dir / image_id, temp_train_img_dir / image_id)
+
+    for image_id in tqdm(test_df["id"], desc=f"Copying test images to {public_dir.name}", total=len(test_df)):
+        shutil.copy(raw_image_dir / image_id, temp_test_img_dir / image_id)
+
+    # and then recompress
+    compress(temp_train_img_dir, public_dir / "train.zip")
+    compress(temp_test_img_dir, public_dir / "test.zip")
+
+    # and cleanup temporary image directories
+    shutil.rmtree(temp_train_img_dir)
+    shutil.rmtree(temp_test_img_dir)
+
+    # checks for the current split
+    assert (public_dir / "train.zip").exists(), f"{public_dir.name}/train.zip should exist"
+    assert (public_dir / "test.zip").exists(), f"{public_dir.name}/test.zip should exist"
+
+    assert len(train_df) + len(test_df) == len(
+        original_df
+    ), "The lengths of the splits should add up to the original"
+    assert len(train_df) > len(test_df), "The train set should be larger than the test set"
+    assert len(test_df) == len(
+        sample_submission
+    ), "The test set should match the sample submission"
+
+    assert (
+        train_df.columns.tolist()
+        == test_df.columns.tolist()
+        == original_df.columns.tolist()
+        == sample_submission.columns.tolist()
+    ), "All dataframes should have the same columns, i.e. ['id', 'has_cactus']"
+
+    assert set(train_df["id"]).isdisjoint(test_df["id"]), "Train and test ids should not overlap"
+    assert set(test_df["id"]) == set(
+        sample_submission["id"]
+    ), "Test and sample_submission ids should match"
+
+    assert train_df["id"].nunique() == len(train_df), "There should be no duplicate ids in train"
+    assert test_df["id"].nunique() == len(test_df), "There should be no duplicate ids in test"
+
+
+def prepare(raw: Path, public: Path, private: Path):
+
+    old_train = read_csv(raw / "train.csv")
+    
+    # need to split the train.zip into train.zip and test.zip; to do so need to extract first
+    extract(raw / "train.zip", raw)
+    raw_image_dir = raw / "train"
+
+    # --- 1. Original Split: Create main train and test sets ---
+    # This block creates the original public/private outputs, which must remain unchanged.
+    
+    # 4000 / (4000 + 17500) -> test_size is ~0.19
+    new_train, new_test = train_test_split(old_train, test_size=0.19, random_state=0)
+    
+    _process_and_save_split(
+        train_df=new_train,
+        test_df=new_test,
+        public_dir=public,
+        private_dir=private,
+        raw_image_dir=raw_image_dir,
+        original_df=old_train,
+    )
+    
+    # --- 2. New Validation Split: Create a validation set from the main train set ---
+    # This block creates the new public_val/private_val outputs.
+    
+    # Define paths for the new validation split artifacts
+    public_val = public.parent / "public_val"
+    private_val = private.parent / "private_val"
+
+    # To make the new validation set (test_val) have the same size as the original
+    # test set (new_test), we calculate the required test_size proportion.
+    test_size_for_val_split = len(new_test) / len(new_train)
+    
+    # Split the main training set (new_train) into a smaller training set (train_val)
+    # and a validation set (test_val), using the same random_state for consistency.
+    train_val, test_val = train_test_split(new_train, test_size=test_size_for_val_split, random_state=0)
+
+    _process_and_save_split(
+        train_df=train_val,
+        test_df=test_val,
+        public_dir=public_val,
+        private_dir=private_val,
+        raw_image_dir=raw_image_dir,
+        original_df=new_train,
+    )
+
+    # --- Final Cleanup ---
+    # and cleanup the extracted raw image directory, which is no longer needed
+    shutil.rmtree(raw_image_dir)
+    assert not (raw / "train").exists(), "raw/train/ should not exist"

mlebench/competitions/alaska2-image-steganalysis/grade.py

@@ -0,0 +1,136 @@
+import numpy as np
+import pandas as pd
+from sklearn import metrics
+
+from mlebench.grade_helpers import InvalidSubmissionError
+
+
+def insert_thresholds(fpr: np.ndarray, tpr: np.ndarray, tpr_thresholds: list) -> tuple:
+    """
+    Insert tpr_thresholds into the TPR and FPR arrays to ensure that the thresholds are present
+    in the TPR array and the corresponding FPR values are interpolated.
+
+    e.g.
+    > tpr = [0.0, 0.25, 0.5, 0.75, 1.0]
+    > fpr = [0.0, 0.1, 0.2, 0.3, 0.4]
+
+    > tpr_thresholds = [0.0, 0.3, 0.4, 1.0]
+    > fpr, tpr = insert_thresholds(fpr, tpr, tpr_thresholds)
+
+    > print(tpr)
+    > print(fpr)
+    [0.0, 0.25, 0.3, 0.4, 0.5, 0.75, 1.0]
+    [0.0, 0.1, 0.12, 0.16, 0.2, 0.3, 0.4]
+    """
+    fpr_ = fpr.tolist().copy()  # Don't modify the input arrays
+    tpr_ = tpr.tolist().copy()
+    for threshold in tpr_thresholds:
+        if threshold not in tpr_:
+            # Find the right position within tpr to insert the threshold
+            for i, tpr_val in enumerate(tpr_):
+                if tpr_val > threshold:
+                    # Linear interpolation of fpr
+                    new_fpr = fpr_[i - 1] + (fpr_[i] - fpr_[i - 1]) * (threshold - tpr_[i - 1]) / (
+                        tpr_[i] - tpr_[i - 1]
+                    )
+
+                    tpr_.insert(i, threshold)
+                    fpr_.insert(i, new_fpr)
+                    break
+    return np.array(fpr_), np.array(tpr_)
+
+
+def alaska_weighted_auc(y_true: np.ndarray, y_pred: np.ndarray) -> float:
+    """
+    Instead of the standard AUC, the competition uses a weighted AUC where different regions of
+    the ROC curve are weighted differently.
+
+    We compute the area under the curve segment by segment (HORIZONTAL segments between TPR pairs
+    on the y-axis), and then compute a weighted average of the segments.
+
+    For more details, see:
+    www.kaggle.com/competitions/alaska2-image-steganalysis/overview/evaluation
+
+    This particular implementation is adapted from:
+    https://www.kaggle.com/code/anokas/weighted-auc-metric-updated
+    (Key change vs the above implementation: The above implementation uses the `tpr` and `fpr`
+    arrays from metrics.roc_curve as is, neglecting to handle the case where the `tpr` values
+    don't line up nicely with the thresholds - leading to a situation where some segments either
+    belong partially to the wrong threshold, or get skipped entirely. Our implementation fixes
+    this by inserting the thresholds into the `tpr` and `fpr` arrays before computing the AUC.)
+    """
+    tpr_thresholds = [0.0, 0.4, 1.0]
+    weights = [2, 1]
+
+    fpr, tpr, thresholds = metrics.roc_curve(y_true, y_pred, pos_label=1)
+    fpr, tpr = insert_thresholds(fpr, tpr, tpr_thresholds)
+
+    # size of subsets
+    areas = np.array(tpr_thresholds[1:]) - np.array(tpr_thresholds[:-1])
+
+    competition_metric = 0
+    # Compute AUC segment by segment (where each segment is a horizontal slice of the ROC curve
+    # between a pair of consecutive TPR thresholds)
+    for idx, weight in enumerate(weights):
+        y_min = tpr_thresholds[idx]
+        y_max = tpr_thresholds[idx + 1]
+
+        # Here, we're creating new x and y arrays to calculate the AUC for this segment:
+
+        # The segment arrays consist of the FPR and TPR values in the segment,
+        mask = (y_min <= tpr) & (tpr <= y_max)
+        if mask.sum() == 0:
+            continue
+        xs = fpr[mask]
+        ys = tpr[mask]
+
+        # plus a new point [1, y_max] which just closes the shape of this segment (draws a
+        # horizontal line from the highest point in this segment to the right of the x-axis)
+        xs = np.concatenate([xs, [1]])
+        ys = np.concatenate([ys, [y_max]])
+        ys = ys - y_min  # normalize such that curve starts at y=0
+        score = metrics.auc(xs, ys)
+        submetric = score * weight
+        competition_metric += submetric
+
+    # The total area is normalized by the sum of weights such that the final weighted AUC is between 0 and 1.
+    normalization = np.dot(areas, weights)
+
+    return competition_metric / normalization
+
+
+def prepare_for_metric(submission: pd.DataFrame, answers: pd.DataFrame):
+    if len(submission) != len(answers):
+        raise InvalidSubmissionError("Submission and answers must have the same length")
+
+    if "Id" not in submission.columns:
+        raise InvalidSubmissionError("Submission must have an 'Id' column")
+
+    if "Label" not in submission.columns:
+        raise InvalidSubmissionError("Submission must have a 'Label' column")
+
+    submission = submission.sort_values("Id")
+    answers = answers.sort_values("Id")
+
+    if (submission["Id"].values != answers["Id"].values).any():
+        raise InvalidSubmissionError("Submission and answers must have the same ids")
+
+    # Answers and submission must be numbers
+    try:
+        submission["Label"] = submission["Label"].astype(float)
+    except ValueError:
+        raise InvalidSubmissionError("Labels in submission must be numbers")
+    answers["Label"] = answers["Label"].astype(float)
+
+    # Cannot contain NaNs
+    assert not answers["Label"].isnull().any(), "Answers cannot contain NaNs"
+    if submission["Label"].isnull().any():
+        raise InvalidSubmissionError("Submission cannot contain NaNs")
+
+    return answers["Label"], submission["Label"]
+
+
+def grade(submission: pd.DataFrame, answers: pd.DataFrame) -> float:
+    y_true, y_pred = prepare_for_metric(submission, answers)
+
+    return alaska_weighted_auc(np.array(y_true), np.array(y_pred))