dslighting 1.7.1__py3-none-any.whl → 1.7.6__py3-none-any.whl

mlebench/competitions/jigsaw-toxic-comment-classification-challenge/prepare.py

@@ -0,0 +1,42 @@
+from pathlib import Path
+
+from mlebench.utils import extract, read_csv
+
+from .classes import CLASSES
+
+
+def prepare(raw: Path, public: Path, private: Path):
+    # extract only what is needed
+    extract(raw / "train.csv.zip", raw)
+    extract(raw / "test.csv.zip", raw)
+    extract(raw / "test_labels.csv.zip", raw)
+    extract(raw / "sample_submission.csv.zip", raw)
+
+    # the test set is provided, so we dont have to split the train set nor form the sample submission
+    train_with_labels = read_csv(raw / "train.csv")
+    test_without_labels = read_csv(raw / "test.csv")
+    answers = read_csv(raw / "test_labels.csv")
+    sample_submission = read_csv(raw / "sample_submission.csv")
+    sample_submission[CLASSES] = 0.5
+
+    # save to public
+    train_with_labels.to_csv(public / "train.csv", index=False)
+    test_without_labels.to_csv(public / "test.csv", index=False)
+    sample_submission.to_csv(public / "sample_submission.csv", index=False)
+
+    # save to private
+    answers.to_csv(private / "test.csv", index=False)
+
+    assert len(answers) == len(
+        sample_submission
+    ), "Private test set and sample submission should be of the same length"
+
+    assert sorted(answers["id"]) == sorted(
+        test_without_labels["id"]
+    ), "Private and Public test IDs should match"
+    assert sorted(sample_submission["id"]) == sorted(
+        test_without_labels["id"]
+    ), "Public test and sample submission IDs should match"
+    assert (
+        len(set(train_with_labels["id"]) & set(test_without_labels["id"])) == 0
+    ), "Train and test IDs should not overlap"

mlebench/competitions/jigsaw-toxic-comment-classification-challenge/prepare_val.py

@@ -0,0 +1,88 @@
+from pathlib import Path
+
+from sklearn.model_selection import train_test_split
+
+from mlebench.utils import extract, read_csv
+
+from .classes import CLASSES
+
+
+def prepare(raw: Path, public: Path, private: Path):
+    # extract only what is needed
+    extract(raw / "train.csv.zip", raw)
+    extract(raw / "test.csv.zip", raw)
+    extract(raw / "test_labels.csv.zip", raw)
+    extract(raw / "sample_submission.csv.zip", raw)
+
+    # the test set is provided, so we dont have to split the train set nor form the sample submission
+    train_with_labels = read_csv(raw / "train.csv")
+    test_without_labels = read_csv(raw / "test.csv")
+    answers = read_csv(raw / "test_labels.csv")
+    sample_submission = read_csv(raw / "sample_submission.csv")
+    sample_submission[CLASSES] = 0.5
+
+    # save to public
+    train_with_labels.to_csv(public / "train.csv", index=False)
+    test_without_labels.to_csv(public / "test.csv", index=False)
+    sample_submission.to_csv(public / "sample_submission.csv", index=False)
+
+    # save to private
+    answers.to_csv(private / "test.csv", index=False)
+
+    assert len(answers) == len(
+        sample_submission
+    ), "Private test set and sample submission should be of the same length"
+
+    assert sorted(answers["id"]) == sorted(
+        test_without_labels["id"]
+    ), "Private and Public test IDs should match"
+    assert sorted(sample_submission["id"]) == sorted(
+        test_without_labels["id"]
+    ), "Public test and sample submission IDs should match"
+    assert (
+        len(set(train_with_labels["id"]) & set(test_without_labels["id"])) == 0
+    ), "Train and test IDs should not overlap"
+
+    # ==================================================================
+    # === New code for creating the validation set starts here ===
+    # The code above this line is untouched to ensure original outputs
+    # remain identical.
+    # ==================================================================
+
+    # 1. Define new paths and create the directories for the validation set.
+    public_val = public.parent / "public_val"
+    private_val = private.parent / "private_val"
+    public_val.mkdir(exist_ok=True)
+    private_val.mkdir(exist_ok=True)
+
+    # 2. Split the original training data to create a new, smaller training set
+    # and a new validation set. The size of the validation set will be the same
+    # as the size of the original test set.
+    validation_set_size = len(test_without_labels)
+    train_val, test_val_with_labels = train_test_split(
+        train_with_labels,
+        test_size=validation_set_size,
+        random_state=42,  # Use a fixed random state for reproducibility
+    )
+
+    # 3. Prepare the validation set files, mirroring the original test set structure.
+    # The public part (input features, without labels)
+    test_val_without_labels = test_val_with_labels[["id", "comment_text"]].copy()
+
+    # The private part (ground truth labels for the validation set)
+    answers_val = test_val_with_labels[["id"] + CLASSES].copy()
+
+    # 4. Create a sample submission file for the new validation set,
+    # mirroring the original sample submission format.
+    sample_submission_val = test_val_without_labels[["id"]].copy()
+    sample_submission_val[CLASSES] = 0.5
+
+    # 5. Save the new sets to the 'public_val' and 'private_val' directories,
+    # using the same filenames as in the original 'public' and 'private' dirs.
+    # Save to public_val
+    train_val.to_csv(public_val / "train.csv", index=False)
+    test_val_without_labels.to_csv(public_val / "test.csv", index=False)
+    sample_submission_val.to_csv(public_val / "sample_submission.csv", index=False)
+
+    # Save to private_val
+    answers_val.to_csv(private_val / "test.csv", index=False)

mlebench/competitions/jigsaw-unintended-bias-in-toxicity-classification/grade.py

@@ -0,0 +1,153 @@
+from typing import Tuple
+
+from pandas import DataFrame
+from sklearn.metrics import roc_auc_score
+
+from mlebench.grade_helpers import InvalidSubmissionError
+
+
+def _filter_for_subgroup(
+    answers: DataFrame, submission: DataFrame, identity_column: str, target_value: int = 1
+) -> Tuple[DataFrame, DataFrame]:
+    """
+    Restricts the data to only the examples that mention the specific identity subgroup.
+    A low value in this metric means the model does a poor job of distinguishing between toxic and non-toxic comments that mention the identity.
+    """
+
+    filtered_answers = answers[answers[identity_column] == target_value]
+    filtered_submission = submission[submission["id"].isin(filtered_answers["id"])]
+    return filtered_answers, filtered_submission
+
+
+def _filter_for_bpsn(
+    answers: DataFrame, submission: DataFrame, identity_column: str, target_value: int = 1
+) -> Tuple[DataFrame, DataFrame]:
+    """
+    Filter for BPSN (Background Positive, Subgroup Negative) AUC
+    Restricts the data to only the non-toxic examples that mention the identity and the toxic examples that do not.
+    A low value in this metric means that the model confuses non-toxic examples that mention the identity with toxic examples that do not, likely meaning that the model predicts higher toxicity scores than it should for non-toxic examples mentioning the identity.
+    """
+    filtered_answers = answers[
+        ((answers["toxicity"] == 0) & (answers[identity_column] == target_value))
+        | ((answers["toxicity"] == 1) & (answers[identity_column] != target_value))
+    ]
+    filtered_submission = submission[submission["id"].isin(filtered_answers["id"])]
+    return filtered_answers, filtered_submission
+
+
+def _filter_for_bnsp(
+    answers: DataFrame, submission: DataFrame, identity_column: str, target_value: int = 1
+) -> Tuple[DataFrame, DataFrame]:
+    """
+    Filter for BNSP (Background Negative, Subgroup Positive) AUC
+    Restricts the data to only the toxic examples that mention the identity and the non-toxic examples that do not.
+    A low value in this metric means that the model confuses toxic examples that mention the identity with non-toxic examples that do not, likely meaning that the model predicts lower toxicity scores than it should for toxic examples mentioning the identity.
+    """
+    filtered_answers = answers[
+        ((answers["toxicity"] == 1) & (answers[identity_column] == target_value))
+        | ((answers["toxicity"] == 0) & (answers[identity_column] != target_value))
+    ]
+    filtered_submission = submission[submission["id"].isin(filtered_answers["id"])]
+    return filtered_answers, filtered_submission
+
+
+def _generalized_mean(scores: list, p: int, identity_columns: list[str]) -> float:
+    return (sum([s**p for s in scores]) / len(identity_columns)) ** (1 / p)
+
+
+def prepare_for_metric(
+    submission: DataFrame, answers: DataFrame, identity_columns: list[str]
+) -> Tuple[DataFrame, DataFrame]:
+
+    # Ensure that the submission DataFrame contains the necessary columns
+    if "id" not in submission.columns:
+        raise InvalidSubmissionError("Submission DataFrame must have an 'id' column.")
+    if (
+        "prediction" not in submission.columns and "toxicity" not in submission.columns
+    ):  # Allow for the submission to be the CSV of answers, which has a "toxicity" column instead of a "prediction" column
+        raise InvalidSubmissionError("Submission DataFrame must have a 'prediction' column.")
+
+    # Ensure that the answers DataFrame contains the necessary columns
+    assert "id" in answers.columns, "Answers DataFrame must have an 'id' column."
+    assert "toxicity" in answers.columns, "Answers DataFrame must have a 'toxicity' column."
+
+    submission = submission.sort_values("id")
+    answers = answers.sort_values("id")
+
+    if (submission["id"].values != answers["id"].values).any():
+        raise InvalidSubmissionError("IDs in the submission and answers DataFrames do not match.")
+
+    # Ensure that all identity columns are present in the answers DataFrame
+    for identity_column in identity_columns:
+        assert (
+            identity_column in answers.columns
+        ), f"Answers DataFrame must have an '{identity_column}' column."
+
+    submission.rename(columns={"prediction": "toxicity"}, inplace=True)
+
+    # Convert continuous toxicity scores to binary toxicity labels
+    submission["toxicity"] = submission["toxicity"].apply(lambda x: 1 if x >= 0.5 else 0)
+    answers["toxicity"] = answers["toxicity"].apply(lambda x: 1 if x >= 0.5 else 0)
+
+    # Convert continuous identity columns to binary identity columns
+    for identity_column in identity_columns:
+        answers[identity_column] = answers[identity_column].apply(lambda x: 1 if x >= 0.5 else 0)
+
+    return submission, answers
+
+
+def grade(
+    submission: DataFrame,
+    answers: DataFrame,
+    identity_columns: list[str] = [
+        "male",
+        "female",
+        "homosexual_gay_or_lesbian",
+        "christian",
+        "jewish",
+        "muslim",
+        "black",
+        "white",
+        "psychiatric_or_mental_illness",
+    ],
+    p: int = -5,
+    w: float = 0.25,
+) -> float:
+    """
+    Computes the custom metric for Jigsaw Unintended Bias in Toxicity Classification.
+    See https://www.kaggle.com/competitions/jigsaw-unintended-bias-in-toxicity-classification/overview/evaluation for details on this metric.
+    """
+
+    submission, answers = prepare_for_metric(submission, answers, identity_columns)
+
+    overall_auc = roc_auc_score(answers["toxicity"], submission["toxicity"])
+
+    # To measure unintended bias, we again calculate the ROC-AUC, this time on three specific subsets of the test set for each identity, each capturing a different aspect of unintended bias.
+    bias_aucs = []
+    for identity_column in identity_columns:
+        subgroup_answers, subgroup_submission = _filter_for_subgroup(
+            answers, submission, identity_column
+        )
+        bpsn_answers, bpsn_submission = _filter_for_bpsn(answers, submission, identity_column)
+        bnsp_answers, bnsp_submission = _filter_for_bnsp(answers, submission, identity_column)
+
+        subgroup_aucs = roc_auc_score(subgroup_answers["toxicity"], subgroup_submission["toxicity"])
+        bpsn_aucs = roc_auc_score(bpsn_answers["toxicity"], bpsn_submission["toxicity"])
+        bnsp_aucs = roc_auc_score(bnsp_answers["toxicity"], bnsp_submission["toxicity"])
+
+        bias_aucs.append([subgroup_aucs, bpsn_aucs, bnsp_aucs])
+
+    # Calculate generalized mean for each identity column
+    generalized_means = [
+        _generalized_mean([bias_auc[i] for bias_auc in bias_aucs], p, identity_columns)
+        for i in range(3)  # there are 3 bias metrics
+    ]
+
+    # Combine the overall AUC with the generalized mean of the Bias AUCs
+    weighted_overall_auc = w * overall_auc
+    weighted_generalized_means = sum(
+        [w * generalized_means[i] for i in range(len(generalized_means))]
+    )
+    final_score = weighted_overall_auc + weighted_generalized_means
+
+    return final_score

mlebench/competitions/jigsaw-unintended-bias-in-toxicity-classification/prepare.py

@@ -0,0 +1,36 @@
+import shutil
+from pathlib import Path
+
+from mlebench.utils import read_csv
+
+
+def prepare(raw: Path, public: Path, private: Path):
+    # Create test and test without labels
+    test = read_csv(raw / "test_private_expanded.csv")
+    test_without_labels = test[["id", "comment_text"]]
+
+    # Copy over files
+    shutil.copy(raw / "test_private_expanded.csv", private / "test.csv")
+    shutil.copy(raw / "train.csv", public / "train.csv")
+    test_without_labels.to_csv(public / "test.csv", index=False)
+
+    # Create a sample submission file
+    submission_df = test_without_labels.copy()
+    submission_df = submission_df.drop(columns=["comment_text"])
+    submission_df["prediction"] = 0.0
+    submission_df.to_csv(public / "sample_submission.csv", index=False)
+
+    # Checks
+    assert test.shape == (
+        len(test),
+        45,
+    ), "test.csv should have 45 columns as per raw data"
+    assert test_without_labels.shape == (
+        len(test_without_labels),
+        2,
+    ), "test.csv should have 2 columns: id, comment_text"
+
+    assert submission_df.shape == (
+        len(test),
+        2,
+    ), "sample_submission.csv should have 2 columns: id, prediction"

mlebench/competitions/jigsaw-unintended-bias-in-toxicity-classification/prepare_val.py

@@ -0,0 +1,117 @@
+import shutil
+from pathlib import Path
+
+# New import required for the data splitting
+from sklearn.model_selection import train_test_split
+
+from mlebench.utils import read_csv
+
+
+def prepare(raw: Path, public: Path, private: Path):
+    """
+    Prepares the data for the competition.
+
+    This script performs two main operations:
+    1.  Processes the original raw data into `public` and `private` directories
+        for the official test set, ensuring the output is identical to the
+        original script's output.
+    2.  Creates a new validation split from the original training data. It splits
+        the original `train.csv` into a new, smaller training set (`train_val`)
+        and a validation set (`test_val`). The results are saved in parallel
+        `public_val` and `private_val` directories with a mirrored file structure.
+    """
+    # --- Part 1: Original Data Preparation (UNCHANGED OUTPUT) ---
+    # This block ensures the 'public' and 'private' directories are populated
+    # exactly as they were in the original script.
+
+    # Ensure original output directories exist
+    public.mkdir(parents=True, exist_ok=True)
+    private.mkdir(parents=True, exist_ok=True)
+
+    # Create test and test without labels
+    test = read_csv(raw / "test_private_expanded.csv")
+    test_without_labels = test[["id", "comment_text"]]
+
+    # Copy over files
+    shutil.copy(raw / "test_private_expanded.csv", private / "test.csv")
+    shutil.copy(raw / "train.csv", public / "train.csv")
+    test_without_labels.to_csv(public / "test.csv", index=False)
+
+    # Create a sample submission file
+    submission_df = test_without_labels.copy()
+    submission_df = submission_df.drop(columns=["comment_text"])
+    submission_df["prediction"] = 0.0
+    submission_df.to_csv(public / "sample_submission.csv", index=False)
+
+    # Checks
+    assert test.shape == (
+        len(test),
+        45,
+    ), "test.csv should have 45 columns as per raw data"
+    assert test_without_labels.shape == (
+        len(test_without_labels),
+        2,
+    ), "test.csv should have 2 columns: id, comment_text"
+
+    assert submission_df.shape == (
+        len(test),
+        2,
+    ), "sample_submission.csv should have 2 columns: id, prediction"
+
+    # --- Part 2: New Validation Set Creation ---
+    # This block creates a new split from the original training data to form
+    # a new, smaller training set and a validation set. Outputs are saved
+    # to 'public_val' and 'private_val' directories.
+
+    # Define and create the new parallel directories for the validation split
+    public_val = public.parent / "public_val"
+    private_val = private.parent / "private_val"
+    public_val.mkdir(parents=True, exist_ok=True)
+    private_val.mkdir(parents=True, exist_ok=True)
+
+    # Load the full original training data, which will be split
+    full_train_df = read_csv(raw / "train.csv")
+
+    # The size of the new validation set ('test_val') should be the same
+    # as the size of the original test set to replicate the split ratio.
+    test_set_size = len(test)
+
+    # Split the full training data into a new training and validation set.
+    # A fixed random_state is used to ensure the split is deterministic.
+    train_val, test_val = train_test_split(
+        full_train_df, test_size=test_set_size, random_state=42
+    )
+
+    # Create the public version of the validation set (without labels)
+    test_val_without_labels = test_val[["id", "comment_text"]]
+
+    # Save the new split data, mirroring the original file structure and names.
+    # private_val contains the ground truth for the new validation set
+    test_val.to_csv(private_val / "test.csv", index=False)
+
+    # public_val contains the new smaller training set and the unlabeled validation set
+    train_val.to_csv(public_val / "train.csv", index=False)
+    test_val_without_labels.to_csv(public_val / "test.csv", index=False)
+
+    # Create a sample submission file for the new validation set
+    submission_df_val = test_val_without_labels.copy()
+    submission_df_val = submission_df_val.drop(columns=["comment_text"])
+    submission_df_val["prediction"] = 0.0
+    submission_df_val.to_csv(public_val / "sample_submission.csv", index=False)
+
+    # Add checks for the new validation split dataframes
+    assert test_val.shape == (
+        len(test_val),
+        full_train_df.shape[1],
+    ), f"private_val/test.csv should have {full_train_df.shape[1]} columns"
+    assert test_val_without_labels.shape == (
+        len(test_val_without_labels),
+        2,
+    ), "public_val/test.csv should have 2 columns: id, comment_text"
+    assert submission_df_val.shape == (
+        len(test_val),
+        2,
+    ), "public_val/sample_submission.csv should have 2 columns: id, prediction"
+    assert len(train_val) + len(test_val) == len(
+        full_train_df
+    ), "Split should conserve all data from original train set"

mlebench/competitions/kuzushiji-recognition/grade.py

@@ -0,0 +1,58 @@
+import pandas as pd
+
+from mlebench.grade_helpers import InvalidSubmissionError
+
+from .kuzushiji_metric import kuzushiji_f1
+
+
+def prepare_for_metric(
+    submission: pd.DataFrame, answers: pd.DataFrame
+) -> tuple[pd.DataFrame, pd.DataFrame]:
+    """
+    No changes needed, since kuzushiji_metric.kuzushiji_f1 expects the submission and answers
+    exactly as they are provided. In this function we just check for invalid submissions.
+    """
+    if len(submission) != len(answers):
+        raise InvalidSubmissionError(
+            f"Submission has {len(submission)} rows, but expected {len(answers)} rows."
+        )
+    if not {"image_id", "labels"}.issubset(submission.columns):
+        raise InvalidSubmissionError("Submission must have columns 'image_id' and 'labels'.")
+
+    submission = submission.sort_values("image_id").reset_index(drop=True)
+    answers = answers.sort_values("image_id").reset_index(drop=True)
+
+    if not all(submission["image_id"] == answers["image_id"]):
+        raise InvalidSubmissionError("Submission image_id codes don't match solution.")
+
+    # Check that labels have the correct format
+    for pred in submission["labels"]:
+        # Skip empty predictions
+        if pd.isna(pred):
+            continue
+
+        # Pred is a string of the form "label1 X1 Y1 label2 X2 Y2 ..."
+        # where label must be a str and X, Y are floats
+        preds = pred.split(" ")
+        if len(preds) % 3 != 0:
+            raise InvalidSubmissionError(f"Malformed prediction string: {pred}")
+        # Check that the X and Y values can be converted to floats
+        if not all(isinstance(float(preds[i]), float) for i in range(1, len(preds), 3)) or not all(
+            isinstance(float(preds[i]), float) for i in range(2, len(preds), 3)
+        ):
+            raise InvalidSubmissionError(f"X and Y values must be floats: {pred}")
+
+    return submission, answers
+
+
+def grade(submission: pd.DataFrame, answers: pd.DataFrame) -> float:
+    """
+    Submissions are evaluated on a modified version of the F1 Score. To score a true positive,
+    you must provide center point coordinates that are within the ground truth bounding box and
+    a matching label. The ground truth bounding boxes are defined in the format {label X Y Width Height},
+    so if the ground truth label is U+003F 1 1 10 10 then a prediction of U+003F 3 3 would pass.
+
+    See https://www.kaggle.com/competitions/kuzushiji-recognition/overview for more details.
+    """
+    submission, answers = prepare_for_metric(submission, answers)
+    return float(kuzushiji_f1(sub=submission, solution=answers))

mlebench/competitions/kuzushiji-recognition/kuzushiji_metric.py

@@ -0,0 +1,118 @@
+# ADAPTED FROM: https://gist.github.com/SohierDane/a90ef46d79808fe3afc70c80bae45972
+"""
+Python equivalent of the Kuzushiji competition metric (https://www.kaggle.com/c/kuzushiji-recognition/)
+Kaggle's backend uses a C# implementation of the same metric. This version is
+provided for convenience only; in the event of any discrepancies the C# implementation
+is the master version.
+
+Tested on Python 3.6 with numpy 1.16.4 and pandas 0.24.2.
+
+Update 2024/06/05: Also tested on Python 3.12 with numpy 1.26.4 and pandas 2.2.2.
+"""
+
+
+import multiprocessing
+
+import numpy as np
+import pandas as pd
+
+
+def score_page(preds, truth):
+    """
+    Scores a single page.
+    Args:
+        preds: prediction string of labels and center points.
+        truth: ground truth string of labels and bounding boxes.
+    Returns:
+        True/false positive and false negative counts for the page
+    """
+    tp = 0
+    fp = 0
+    fn = 0
+
+    truth_indices = {"label": 0, "X": 1, "Y": 2, "Width": 3, "Height": 4}
+    preds_indices = {"label": 0, "X": 1, "Y": 2}
+
+    if pd.isna(truth) and pd.isna(preds):
+        return {"tp": tp, "fp": fp, "fn": fn}
+
+    if pd.isna(truth):
+        fp += len(preds.split(" ")) // len(preds_indices)
+        return {"tp": tp, "fp": fp, "fn": fn}
+
+    if pd.isna(preds):
+        fn += len(truth.split(" ")) // len(truth_indices)
+        return {"tp": tp, "fp": fp, "fn": fn}
+
+    truth = truth.split(" ")
+    if len(truth) % len(truth_indices) != 0:
+        raise ValueError("Malformed solution string")
+    truth_label = np.array(truth[truth_indices["label"] :: len(truth_indices)])
+    truth_xmin = np.array(truth[truth_indices["X"] :: len(truth_indices)]).astype(float)
+    truth_ymin = np.array(truth[truth_indices["Y"] :: len(truth_indices)]).astype(float)
+    truth_xmax = truth_xmin + np.array(truth[truth_indices["Width"] :: len(truth_indices)]).astype(
+        float
+    )
+    truth_ymax = truth_ymin + np.array(truth[truth_indices["Height"] :: len(truth_indices)]).astype(
+        float
+    )
+
+    preds = preds.split(" ")
+    if len(preds) % len(preds_indices) != 0:
+        raise ValueError("Malformed prediction string")
+    preds_label = np.array(preds[preds_indices["label"] :: len(preds_indices)])
+    preds_x = np.array(preds[preds_indices["X"] :: len(preds_indices)]).astype(float)
+    preds_y = np.array(preds[preds_indices["Y"] :: len(preds_indices)]).astype(float)
+    preds_unused = np.ones(len(preds_label)).astype(bool)
+
+    for xmin, xmax, ymin, ymax, label in zip(
+        truth_xmin, truth_xmax, truth_ymin, truth_ymax, truth_label
+    ):
+        # Matching = point inside box & character same & prediction not already used
+        matching = (
+            (xmin < preds_x)
+            & (xmax > preds_x)
+            & (ymin < preds_y)
+            & (ymax > preds_y)
+            & (preds_label == label)
+            & preds_unused
+        )
+        if matching.sum() == 0:
+            fn += 1
+        else:
+            tp += 1
+            preds_unused[np.argmax(matching)] = False
+    fp += preds_unused.sum()
+    return {"tp": tp, "fp": fp, "fn": fn}
+
+
+def kuzushiji_f1(sub, solution):
+    """
+    Calculates the competition metric.
+    Args:
+        sub: submissions, as a Pandas dataframe
+        solution: solution, as a Pandas dataframe
+    Returns:
+        f1 score
+    """
+    if not all(sub["image_id"].values == solution["image_id"].values):
+        raise ValueError("Submission image id codes don't match solution")
+
+    pool = multiprocessing.Pool()
+    results = pool.starmap(score_page, zip(sub["labels"].values, solution["labels"].values))
+    pool.close()
+    pool.join()
+
+    tp = sum([x["tp"] for x in results])
+    fp = sum([x["fp"] for x in results])
+    fn = sum([x["fn"] for x in results])
+
+    if (tp + fp) == 0 or (tp + fn) == 0:
+        return 0
+    precision = tp / (tp + fp)
+    recall = tp / (tp + fn)
+    if precision > 0 and recall > 0:
+        f1 = (2 * precision * recall) / (precision + recall)
+    else:
+        f1 = 0
+    return f1