dslighting 1.7.1__py3-none-any.whl → 1.7.8__py3-none-any.whl

mlebench/competitions/tweet-sentiment-extraction/prepare_val.py

@@ -0,0 +1,106 @@
+from pathlib import Path
+import pandas as pd
+
+from sklearn.model_selection import train_test_split
+
+from mlebench.utils import read_csv
+
+
+def _split_and_save(
+    source_df: pd.DataFrame,
+    test_size: float,
+    random_state: int,
+    public_dir: Path,
+    private_dir: Path,
+    label_col: str = "selected_text",
+    id_col: str = "textID",
+) -> tuple[pd.DataFrame, pd.DataFrame]:
+    """
+    Helper function to perform a train/test split and save the results.
+
+    Args:
+        source_df: The DataFrame to split.
+        test_size: The proportion of the dataset to allocate to the test split.
+        random_state: The seed used by the random number generator.
+        public_dir: The directory to save public artifacts (train set, unlabeled test set).
+        private_dir: The directory to save private artifacts (labeled test set).
+        label_col: The name of the column containing the label.
+        id_col: The name of the column containing the unique ID.
+
+    Returns:
+        A tuple containing the created train and test DataFrames.
+    """
+    # Ensure output directories exist
+    public_dir.mkdir(parents=True, exist_ok=True)
+    private_dir.mkdir(parents=True, exist_ok=True)
+
+    # Perform the split
+    train_df, test_df = train_test_split(
+        source_df, test_size=test_size, random_state=random_state
+    )
+    test_df_without_labels = test_df.drop(columns=[label_col])
+
+    # Save the datasets
+    train_df.to_csv(public_dir / "train.csv", index=False)
+    test_df.to_csv(private_dir / "test.csv", index=False)
+    test_df_without_labels.to_csv(public_dir / "test.csv", index=False)
+
+    assert len(test_df_without_labels) == len(
+        test_df
+    ), f"Expected test_df_without_labels ({len(test_df_without_labels)}) == test_df ({len(test_df)})"
+    assert len(train_df) + len(test_df) == len(
+        source_df
+    ), f"Expected train_df ({len(train_df)}) + test_df ({len(test_df)}) == source_df ({len(source_df)})"
+
+    # Create and save a sample submission file
+    submission_df = test_df.copy()[[id_col, label_col]]
+    submission_df[label_col] = ""
+
+    submission_df.to_csv(public_dir / "sample_submission.csv", index=False)
+    assert len(submission_df) == len(
+        test_df
+    ), f"Expected submission_df ({len(submission_df)}) == test_df ({len(test_df)})"
+
+    return train_df, test_df
+
+
+def prepare(raw: Path, public: Path, private: Path):
+    """
+    Splits the data in raw into public and private datasets with appropriate test/train splits.
+    Also creates a second, parallel split for validation purposes.
+    """
+    old_train = read_csv(raw / "train.csv")
+
+    # ---- Original Data Split (for final test set) ----
+    # This split creates the main train and test sets for the competition.
+    # The outputs in `public` and `private` are left completely unchanged.
+    train_for_val_split, _ = _split_and_save(
+        source_df=old_train,
+        test_size=0.1,
+        random_state=0,
+        public_dir=public,
+        private_dir=private,
+    )
+
+    # ---- New Validation Data Split ----
+    # This second split takes the training set created above and splits it *again*
+    # to create a new, smaller training set and a validation set. The results are
+    # saved to parallel `public_val` and `private_val` directories.
+
+    # Define paths for the new validation split
+    public_val = public.parent / "public_val"
+    private_val = private.parent / "private_val"
+
+    # The original test set size was 0.1 of the total data. The new train set
+    # is 0.9 of the total. To make the new validation set (test_val) have
+    # roughly the same number of samples as the original test set, we use a
+    # test_size of (0.1 / 0.9) = 1/9.
+    validation_test_size = 1 / 9
+
+    _split_and_save(
+        source_df=train_for_val_split,
+        test_size=validation_test_size,
+        random_state=0,  # Use same random_state for consistency
+        public_dir=public_val,
+        private_dir=private_val,
+    )

mlebench/competitions/us-patent-phrase-to-phrase-matching/grade.py

@@ -0,0 +1,31 @@
+import pandas as pd
+from scipy.stats import pearsonr
+
+from mlebench.grade_helpers import InvalidSubmissionError
+
+
+def prepare_for_metric(submission: pd.DataFrame, answers: pd.DataFrame) -> tuple:
+    """
+    `submission` and `answers` are pd.DataFrames with columns `id` and `score`
+    Submissions are evaluated on the Pearson correlation coefficient between the predicted and actual similarity scores.
+    """
+    if len(submission) != len(answers):
+        raise InvalidSubmissionError("Submission and answers must have the same number of rows")
+    if not set(["id", "score"]).issubset(submission.columns):
+        raise InvalidSubmissionError(
+            f"Submission must have columns ['id', 'score'], got {submission.columns}"
+        )
+
+    submission = submission.sort_values("id")
+    answers = answers.sort_values("id")
+    if (submission["id"].values != answers["id"].values).any():
+        raise InvalidSubmissionError("Ids mismatch between submission and answers")
+    y_true = answers["score"].to_numpy()
+    y_pred = submission["score"].to_numpy()
+
+    return y_true, y_pred
+
+
+def grade(submission: pd.DataFrame, answers: pd.DataFrame) -> float:
+    y_true, y_pred = prepare_for_metric(submission, answers)
+    return pearsonr(y_true, y_pred)[0]

mlebench/competitions/us-patent-phrase-to-phrase-matching/prepare.py

@@ -0,0 +1,33 @@
+from pathlib import Path
+
+from sklearn.model_selection import train_test_split
+
+from mlebench.utils import read_csv
+
+
+def prepare(raw: Path, public: Path, private: Path):
+    """
+    Splits the data in raw into public and private datasets with appropriate test/train splits.
+    """
+    public.mkdir(exist_ok=True)
+    private.mkdir(exist_ok=True)
+
+    # Create train, test from train split
+    old_train = read_csv(raw / "train.csv")
+    new_train, new_test = train_test_split(old_train, test_size=0.1, random_state=0)
+    new_test_without_labels = new_test.drop(columns=["score"])
+
+    # Save new train and test
+    new_train.to_csv(public / "train.csv", index=False)
+    new_test.to_csv(private / "test.csv", index=False)
+    new_test_without_labels.to_csv(public / "test.csv", index=False)
+
+    assert len(new_train) + len(new_test) == len(old_train)
+    assert len(new_test) == len(new_test_without_labels)
+
+    # Create a sample submission file
+    submission_df = new_test.copy()[["id", "score"]]
+    submission_df["score"] = 0
+    submission_df.to_csv(public / "sample_submission.csv", index=False)
+
+    assert len(submission_df) == len(new_test)

mlebench/competitions/us-patent-phrase-to-phrase-matching/prepare_val.py

@@ -0,0 +1,71 @@
+from pathlib import Path
+
+from sklearn.model_selection import train_test_split
+
+from mlebench.utils import read_csv
+
+
+def prepare(raw: Path, public: Path, private: Path):
+    """
+    Splits the data in raw into public and private datasets with appropriate test/train splits.
+    Additionally, it creates a secondary validation split for model development.
+    """
+    # Create directories for the primary competition split
+    public.mkdir(exist_ok=True)
+    private.mkdir(exist_ok=True)
+
+    # Create directories for the new validation split
+    public_val = public.parent / "public_val"
+    private_val = private.parent / "private_val"
+    public_val.mkdir(exist_ok=True)
+    private_val.mkdir(exist_ok=True)
+
+    # Load raw data
+    old_train = read_csv(raw / "train.csv")
+
+    # --- Block 1: Original Split (for final competition data) ---
+    # This block creates the main train/test split. Its output in `public`
+    # and `private` must remain unchanged from the original script.
+
+    # Create train, test from the full raw data
+    train, test = train_test_split(old_train, test_size=0.1, random_state=0)
+    test_without_labels = test.drop(columns=["score"])
+
+    # Save new train and test to the original directories
+    train.to_csv(public / "train.csv", index=False)
+    test.to_csv(private / "test.csv", index=False)
+    test_without_labels.to_csv(public / "test.csv", index=False)
+
+    assert len(train) + len(test) == len(old_train)
+    assert len(test) == len(test_without_labels)
+
+    # Create a sample submission file for the original test set
+    submission_df = test.copy()[["id", "score"]]
+    submission_df["score"] = 0
+    submission_df.to_csv(public / "sample_submission.csv", index=False)
+
+    assert len(submission_df) == len(test)
+
+    # --- Block 2: New Validation Split (for model development) ---
+    # This block takes the `train` data from the first split and splits it
+    # again to create a new, smaller training set and a validation set.
+    # The logic and filenames mirror the original split for consistency.
+
+    # Create a new train/validation split from the main training set
+    train_val, test_val = train_test_split(train, test_size=0.1, random_state=0)
+    test_val_without_labels = test_val.drop(columns=["score"])
+
+    # Save the new validation split data into the `_val` directories
+    train_val.to_csv(public_val / "train.csv", index=False)
+    test_val.to_csv(private_val / "test.csv", index=False)
+    test_val_without_labels.to_csv(public_val / "test.csv", index=False)
+
+    assert len(train_val) + len(test_val) == len(train)
+    assert len(test_val) == len(test_val_without_labels)
+
+    # Create a sample submission file for the new validation set
+    submission_val_df = test_val.copy()[["id", "score"]]
+    submission_val_df["score"] = 0
+    submission_val_df.to_csv(public_val / "sample_submission.csv", index=False)
+
+    assert len(submission_val_df) == len(test_val)

mlebench/competitions/utils.py

@@ -0,0 +1,266 @@
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+from numpy import ndarray
+from scipy.special import softmax
+
+from mlebench.grade_helpers import InvalidSubmissionError
+from mlebench.utils import get_logger
+
+logger = get_logger(__name__)
+
+
+def df_to_one_hot(
+    df: pd.DataFrame, id_column: str, target_column: str, classes: list[str]
+) -> pd.DataFrame:
+    """
+    Convert class labels to one-hot encoded vectors.
+    """
+
+    y_onehot = pd.DataFrame(0, index=df.index, columns=[id_column] + classes)
+    y_onehot[id_column] = df[id_column]
+
+    for i, row in df.iterrows():
+        y_onehot.loc[i, row[target_column]] = 1
+
+    return y_onehot
+
+
+def one_hot_dfs_to_log_loss_inputs(
+    submission_one_hot: pd.DataFrame,
+    answers_one_hot: pd.DataFrame,
+    id_column: str = "id",
+    apply_softmax: bool = True,
+) -> dict:
+    """
+    Frequently used logic to prepare one-hotted dfs for log loss calculation.
+    """
+    required_cols = set(answers_one_hot.columns)
+    submission_cols = set(submission_one_hot.columns)
+
+    if not submission_cols.issuperset(required_cols):
+        raise InvalidSubmissionError(
+            f"The submission DataFrame is missing some columns required by the `answers` DataFrame. "
+            f"Missing columns: {required_cols - submission_cols}."
+        )
+    if id_column not in submission_one_hot.columns:
+        raise InvalidSubmissionError(f"Submission is missing id column '{id_column}'.")
+
+    assert id_column in answers_one_hot.columns, f"Answers is missing id column '{id_column}'."
+
+    # Filter submission to only include columns that are in the answers
+    submission_filtered = submission_one_hot[
+        [col for col in answers_one_hot.columns if col in submission_cols]
+    ]
+
+    # Sort submission and answers by id to align them
+    submission_sorted = submission_filtered.sort_values(by=id_column).reset_index(drop=True)
+    answers_sorted = answers_one_hot.sort_values(by=id_column).reset_index(drop=True)
+
+    assert submission_sorted[id_column].tolist() == answers_sorted[id_column].tolist(), (
+        f"Mismatch in {id_column.capitalize()}s between `submission` and `answers` after sorting. "
+        f"Number of mismatched {id_column.capitalize()}s: {len(set(submission_sorted[id_column]) ^ set(answers_sorted[id_column]))}. "
+        f"Ensure both DataFrames have the same {id_column.capitalize()}s."
+    )
+
+    assert list(submission_sorted.columns) == list(answers_sorted.columns), (
+        "Column order mismatch after filtering and sorting. "
+        "Ensure both DataFrames have columns in the same order."
+    )
+
+    y_true = answers_sorted.drop(columns=[id_column]).to_numpy()
+    y_pred = submission_sorted.drop(columns=[id_column]).to_numpy()
+
+    if apply_softmax and is_one_hot_encoded(y_pred):
+        logger.warning(
+            "The flag `apply_softmax` has been set to `True` but the submission is already "
+            "one-hot encoded. Skipping softmax."
+        )
+
+    if apply_softmax and not is_one_hot_encoded(y_pred):
+        y_pred = softmax(y_pred, axis=-1)
+
+    log_loss_inputs = {
+        "y_true": y_true,
+        "y_pred": y_pred,
+    }
+
+    return log_loss_inputs
+
+
+def is_one_hot_encoded(xs: ndarray) -> bool:
+    """Check if a 2D NumPy array is one-hot encoded."""
+
+    assert isinstance(xs, ndarray), f"Expected a NumPy array, got {type(xs)}."
+    assert xs.ndim == 2, f"Expected a 2D array, got {xs.ndim}D."
+
+    is_binary_matrix = np.bitwise_or(xs == 0, xs == 1).all()
+    is_normalized = np.allclose(xs.sum(axis=-1), 1)
+    is_one_hot = bool(is_binary_matrix and is_normalized)
+
+    assert isinstance(is_one_hot, bool), f"Expected a boolean, got {type(is_one_hot)}."
+
+    return is_one_hot
+
+
+def rle_decode(rle_string: str, height: int, width: int) -> ndarray:
+    """
+    Decode an RLE string into a binary mask. The RLE encoding is top-down, left-right. So 1 is
+    (1,1), 2 is (2, 1), etc. The RLE is 1-indexed. Checks that the pairs are sorted, positive, and
+    the decoded pixel values do not overlap.
+
+    Args:
+        rle_string (str): The RLE string.
+        height (int): The height of the image.
+        width (int): The width of the image.
+
+    Returns:
+        np.array: The decoded binary mask.
+    """
+
+    assert isinstance(
+        rle_string, str
+    ), f"Expected a string, but got {type(rle_string)}: {rle_string}"
+    assert isinstance(height, int), f"Expected an integer, but got {type(height)}: {height}"
+    assert isinstance(width, int), f"Expected an integer, but got {type(width)}: {width}"
+
+    if not rle_string.strip():  # Check if the string is empty or contains only whitespace
+        return np.zeros((height, width), dtype=bool)
+
+    s = list(map(int, rle_string.split()))
+    starts, lengths = s[0::2], s[1::2]
+
+    assert starts == sorted(starts), "The pairs in the RLE string must be sorted."
+    assert all(x > 0 for x in starts), "All pairs in the RLE string must be positive integers."
+    assert all(x > 0 for x in lengths), "All pairs in the RLE string must be positive integers."
+
+    # Convert to 0-based indices
+    starts = np.array(starts) - 1
+    ends = starts + lengths
+    img = np.zeros(height * width, dtype=bool)
+
+    for lo, hi in zip(starts, ends):
+        assert not img[lo:hi].any(), "Overlapping RLE pairs are not allowed."
+        img[lo:hi] = True
+
+    # reshape appropriately given how the RLE was encoded
+    return img.reshape((width, height)).T
+
+
+# https://www.kaggle.com/code/inversion/contrails-rle-submission
+def rle_encode(x: ndarray, fg_val=1):
+    """
+    Args:
+        x:  numpy array of shape (height, width), 1 - mask, 0 - background
+    Returns: run length encoding as list
+    """
+    dots = np.where(x.T.flatten() == fg_val)[0]  # .T sets Fortran order down-then-right
+    run_lengths = []
+    prev = -2
+    for b in dots:
+        if b > prev + 1:
+            run_lengths.extend((b + 1, 0))
+        run_lengths[-1] += 1
+        prev = b
+    return run_lengths
+
+
+def rles_to_masks(
+    rl_encodings: list[str], image_heights: list[int], image_widths: list[int]
+) -> list[np.ndarray]:
+    """
+    Performs run-length decoding on a list of run-length encodings to get the binary masks
+    """
+    masks = [
+        rle_decode(encoding, height=image_height, width=image_width)
+        for encoding, image_height, image_width in zip(rl_encodings, image_heights, image_widths)
+    ]
+    return masks
+
+
+def get_ids_from_tf_records(tf_record_path: Path, id_feature: str = "image_name") -> list[str]:
+    import tensorflow as tf  # Import only if needed, otherwise it slows down the module import
+
+    tf_record_dataset = tf.data.TFRecordDataset(tf_record_path.as_posix())
+
+    ids = []
+    for record in tf_record_dataset:
+        features = tf.train.Example.FromString(record.numpy())
+        id = features.features.feature[id_feature].bytes_list.value[0].decode("utf-8")
+        ids.append(id)
+
+    return ids
+
+
+def prepare_for_accuracy_metric(
+    submission: pd.DataFrame,
+    answers: pd.DataFrame,
+    target_column: str,
+    id_column: str,
+) -> dict:
+
+    # Answers checks
+    assert target_column in answers.columns, f"Answers must have a `{target_column}` column"
+    assert id_column in answers.columns, f"Answers must have a `{id_column}` column"
+
+    # Submission checks
+    if len(submission) != len(answers):
+        raise InvalidSubmissionError("Submission must have the same length as the answers.")
+    if target_column not in submission.columns:
+        raise InvalidSubmissionError(f"Submission must have a `{target_column}` column")
+    if id_column not in submission.columns:
+        raise InvalidSubmissionError(f"Submission must have a `{id_column}` column")
+
+    # sort on id to ensure correct order
+    submission = submission.sort_values(id_column)
+    answers = answers.sort_values(id_column)
+
+    if (submission[id_column].values != answers[id_column].values).any():
+        raise InvalidSubmissionError(f"Submission and Answers `{id_column}`'s do not match")
+
+    y_pred = submission[target_column].to_numpy()
+    y_true = answers[target_column].to_numpy()
+
+    return {"y_true": y_true, "y_pred": y_pred}
+
+
+def prepare_for_auroc_metric(
+    submission: pd.DataFrame, answers: pd.DataFrame, id_col: str, target_col: str
+) -> dict:
+
+    # Answers checks
+    assert id_col in answers.columns, f"answers dataframe should have an {id_col} column"
+    assert target_col in answers.columns, f"answers dataframe should have a {target_col} column"
+
+    # Submission checks
+    if id_col not in submission.columns:
+        raise InvalidSubmissionError(f"Submission should have an {id_col} column")
+    if target_col not in submission.columns:
+        raise InvalidSubmissionError(f"Submission should have a {target_col} column")
+    if len(submission) != len(answers):
+        raise InvalidSubmissionError(f"Submission and answers should have the same number of rows")
+    try:
+        pd.to_numeric(submission[target_col])
+    except ValueError:
+        raise InvalidSubmissionError(
+            f"Expected {target_col} column to be numeric, got {submission[target_col].dtype} instead"
+        )
+    if submission[target_col].min() < 0 or submission[target_col].max() > 1:
+        raise InvalidSubmissionError(
+            f"Submission {target_col} column should contain probabilities,"
+            " and therefore contain values between 0 and 1 inclusive"
+        )
+    # Sort
+    submission = submission.sort_values(id_col)
+    answers = answers.sort_values(id_col)
+
+    if (submission[id_col].values != answers[id_col].values).any():
+        raise InvalidSubmissionError(f"Submission and answers should have the same {id_col} values")
+
+    roc_auc_inputs = {
+        "y_true": answers[target_col].to_numpy(),
+        "y_score": submission[target_col].to_numpy(),
+    }
+
+    return roc_auc_inputs

mlebench/competitions/uw-madison-gi-tract-image-segmentation/grade.py

@@ -0,0 +1,158 @@
+import numpy as np
+from pandas import DataFrame
+from scipy.spatial.distance import directed_hausdorff
+from tqdm.auto import tqdm
+
+from mlebench.competitions.utils import rles_to_masks
+from mlebench.grade_helpers import InvalidSubmissionError
+from mlebench.metrics import dice_coefficient
+
+UNIT_CUBE_DIAGONAL = np.sqrt(3)
+
+
+def group_masks_by_day(masks: list[np.ndarray], cases_days_slices: list[str]) -> list[np.ndarray]:
+    """
+    Groups the 2D masks into 3D masks based on the case days
+    - We first join masks of the same slice into a single mask with an OR operation
+    - We then stack them with all the other slices from the same day in the same case into a 3D image
+    """
+    case_day_slice_to_masks = {}
+    unique_slices = []  # list to keep ordering consistent
+    for mask, case_day_slice in sorted(zip(masks, cases_days_slices), key=lambda x: x[1]):
+        if case_day_slice not in case_day_slice_to_masks:
+            case_day_slice_to_masks[case_day_slice] = []
+            unique_slices.append(case_day_slice)
+        case_day_slice_to_masks[case_day_slice].append(mask)
+
+    case_day_to_masks = {}
+    case_days = []  # list to keep ordering consistent
+    for case_day_slice in unique_slices:
+        # go from e.g. "case123_day2_slice_0001" to "case123_day2"
+        case_day = case_day_slice.split("_slice_")[0]
+        if case_day not in case_day_to_masks:
+            case_day_to_masks[case_day] = []
+            case_days.append(case_day)
+        # OR on the masks from the same case_day_slice
+        joined_mask = np.logical_or.reduce(case_day_slice_to_masks[case_day_slice]).astype(np.uint8)
+        # add the joined slice to its relevant case_day
+        case_day_to_masks[case_day].append(joined_mask)
+
+    # now we stack the joined masks for each case_day to make a 3d image
+    for case_day in case_day_to_masks:
+        # make into 3d image
+        case_day_to_masks[case_day] = np.stack(case_day_to_masks[case_day], axis=0)
+
+    # keep the order of the cases consistent
+    case_day_masks = [case_day_to_masks[case_day] for case_day in case_days]
+    return case_day_masks
+
+
+def prepare_for_metric(submission: DataFrame, answers: DataFrame):
+    """
+    Answers dataframe contains image_height and image_width columns necessary for RLE decoding
+    """
+
+    assert "image_height" in answers.columns, "image_height column not found in answers"
+    assert "image_width" in answers.columns, "image_width column not found in answers"
+    if not {"id", "class", "predicted"}.issubset(set(submission.columns)):
+        raise InvalidSubmissionError("Submission must have columns: id, class, predicted")
+
+    # prepare appropriate types
+    submission["predicted"] = submission["predicted"].fillna("")
+    answers["predicted"] = answers["predicted"].fillna("")
+    answers["image_height"] = answers["image_height"].astype(int)
+    answers["image_width"] = answers["image_width"].astype(int)
+
+    image_heights = answers["image_height"].tolist()
+    image_widths = answers["image_width"].tolist()
+
+    # extract masked images from RLE in the dataframes
+    submission_masks = rles_to_masks(submission["predicted"].to_list(), image_heights, image_widths)
+    answer_masks = rles_to_masks(answers["predicted"].to_list(), image_heights, image_widths)
+
+    # group 2d slice masks into 3d case_day masks
+    submission_masks_by_day = group_masks_by_day(submission_masks, submission["id"])
+    answer_masks_by_day = group_masks_by_day(answer_masks, answers["id"])
+
+    return submission_masks, answer_masks, submission_masks_by_day, answer_masks_by_day
+
+
+def hausdorff_distance(predicted_mask: np.ndarray, true_mask: np.ndarray) -> float:
+    """
+    Computes the hausdorff distance between two 3-dimensional binary masks (H, W, C)
+
+    Args:
+        predicted_mask: A 3D binary numpy array indicating where the segmentation is predicted
+        true_mask: A 3D binary numpy array indicating where the segmentation is
+    """
+    # if both empty, return nan, if identical return 0, if only one is empty, return 1
+    if np.sum(predicted_mask) == 0 and np.sum(true_mask) == 0:
+        return np.nan
+    elif np.array_equal(predicted_mask, true_mask):
+        return 0
+    elif (np.sum(predicted_mask) == 0 and np.sum(true_mask) > 0) or (
+        np.sum(predicted_mask) > 0 and np.sum(true_mask) == 0
+    ):
+        return 1
+
+    # if the pred mask is > 1 OOM larger than the true mask, return 1 to avoid unnecess heavy computation
+    if predicted_mask.sum() > 10 * true_mask.sum():
+        return 1
+
+    true_coordinates = np.argwhere(true_mask)
+    predicted_coordinates = np.argwhere(predicted_mask)
+
+    # normalize so that the coordinates are in the range [0, 1], to get a unit cube
+    normalization_constants = true_mask.shape
+    true_coordinates = true_coordinates / normalization_constants
+    predicted_coordinates = predicted_coordinates / normalization_constants
+
+    # Symmetrize Hausdorff distance
+    hausdorff_distance1, _, _ = directed_hausdorff(true_coordinates, predicted_coordinates)
+    hausdorff_distance2, _, _ = directed_hausdorff(predicted_coordinates, true_coordinates)
+    hausdorff_distance = max(hausdorff_distance1, hausdorff_distance2)
+
+    # normalize by the maximum distance possible in a unit cube, which is the diagonal
+    hausdorff_distance = hausdorff_distance / UNIT_CUBE_DIAGONAL
+
+    return hausdorff_distance
+
+
+def grade(submission: DataFrame, answers: DataFrame) -> float:
+    """
+    Computes dice_weight * dice + hausdorff_weight * (1 - hausdorff)
+    """
+    dice_weight = 0.4
+    hausdorff_weight = 0.6
+
+    (
+        submission_masks,
+        answer_masks,
+        submission_masks_by_day,
+        answer_masks_by_day,
+    ) = prepare_for_metric(submission, answers)
+
+    # we skip cases where both masks are empty, so we use nanmean
+    # https://www.kaggle.com/competitions/uw-madison-gi-tract-image-segmentation/discussion/324934#1795540
+    dice_coeff = np.nanmean(
+        [
+            dice_coefficient(submission_mask, answer_mask, np.nan)
+            for submission_mask, answer_mask in tqdm(
+                zip(submission_masks, answer_masks),
+                total=len(submission_masks),
+            )
+        ]
+    )
+
+    # skip cases where both masks are empty, to match behaviour of dice
+    hausdorff_dis = np.nanmean(
+        [
+            hausdorff_distance(submission_mask_by_day, answer_mask_by_day)
+            for submission_mask_by_day, answer_mask_by_day in tqdm(
+                zip(submission_masks_by_day, answer_masks_by_day),
+                total=len(submission_masks_by_day),
+            )
+        ]
+    )
+
+    return dice_weight * dice_coeff + hausdorff_weight * (1 - hausdorff_dis)