dslighting 1.7.1__py3-none-any.whl → 1.7.6__py3-none-any.whl

mlebench/competitions/icecube-neutrinos-in-deep-ice/prepare_val.py

@@ -0,0 +1,183 @@
+import shutil
+from pathlib import Path
+
+import pandas as pd
+from sklearn.model_selection import train_test_split
+from tqdm.auto import tqdm
+
+from mlebench.utils import get_logger
+
+logger = get_logger(__name__)
+
+
+def _create_split_files(
+    original_meta_df: pd.DataFrame,
+    train_batch_ids: list,
+    test_batch_ids: list,
+    public_dir: Path,
+    private_dir: Path,
+    raw_dir: Path,
+):
+    """
+    Helper function to process a split (e.g., train/test or train_val/test_val),
+    save the necessary files, and run checks.
+    """
+    logger.info(f"Processing split for destination: {public_dir.name}")
+    public_dir.mkdir(exist_ok=True, parents=True)
+    private_dir.mkdir(exist_ok=True, parents=True)
+
+    new_train = (
+        original_meta_df[original_meta_df["batch_id"].isin(train_batch_ids)]
+        .reset_index(drop=True)
+        .copy()
+    )
+    new_test = (
+        original_meta_df[original_meta_df["batch_id"].isin(test_batch_ids)]
+        .reset_index(drop=True)
+        .copy()
+    )
+
+    logger.info("Creating label-less test and sample submission")
+    new_test_without_labels = new_test.drop(columns=["azimuth", "zenith"])
+
+    # format private test to only contain event_id and labels
+    new_test = new_test[["event_id", "azimuth", "zenith"]]
+
+    # copy the format as the private test and fill dummy values like kaggle.com
+    sample_submission = new_test.copy()
+    sample_submission["azimuth"] = 1
+    sample_submission["zenith"] = 1
+
+    logger.info("Saving metadata files")
+    new_train.to_parquet(public_dir / "train_meta.parquet", index=False, engine="fastparquet")
+    new_test_without_labels.to_parquet(
+        public_dir / "test_meta.parquet", index=False, engine="fastparquet"
+    )
+    sample_submission.to_csv(public_dir / "sample_submission.csv", index=False)
+    new_test.to_csv(private_dir / "test.csv", index=False)
+
+    logger.info("Copying sensor geometry and batch data files")
+    # sensor_geometry can be copied as is
+    shutil.copy(raw_dir / "sensor_geometry.csv", public_dir / "sensor_geometry.csv")
+
+    # copy the raw train files to train and test folders respectively
+    train_ids_set = set(train_batch_ids)
+    train_dest = public_dir / "train"
+    train_dest.mkdir(exist_ok=True, parents=True)
+    test_ids_set = set(test_batch_ids)
+    test_dest = public_dir / "test"
+    test_dest.mkdir(exist_ok=True, parents=True)
+
+    for batch_file in tqdm(
+        sorted((raw_dir / "train").glob("*.parquet")), desc=f"Copying batches to {public_dir.name}"
+    ):
+        batch_id = int(batch_file.stem.split("_")[-1])
+        if batch_id in train_ids_set:
+            shutil.copy(batch_file, train_dest / batch_file.name)
+        elif batch_id in test_ids_set:
+            shutil.copy(batch_file, test_dest / batch_file.name)
+
+    logger.info("Running checks")
+    assert len(list(public_dir.glob("train/*.parquet"))) == len(
+        train_batch_ids
+    ), "Not all train batches copied"
+    assert len(list(public_dir.glob("test/*.parquet"))) == len(
+        test_batch_ids
+    ), "Not all test batches copied"
+    assert len(train_batch_ids) + len(test_batch_ids) == len(
+        set(train_batch_ids) | set(test_batch_ids)
+    ), "Something went wrong with splitting the batches"
+
+    # Drop the 'split' column for accurate checks, as it's not in the final data
+    original_meta_subset = original_meta_df[
+        original_meta_df["batch_id"].isin(train_batch_ids + test_batch_ids)
+    ]
+    if "split" in original_meta_subset.columns:
+        original_meta_subset = original_meta_subset.drop(columns=["split"])
+
+    assert len(new_train) + len(new_test) == len(
+        original_meta_subset
+    ), "Expected train + test to equal the original data"
+    assert len(sample_submission) == len(
+        new_test
+    ), "Length mismatch between private test and sample submission"
+
+    assert sample_submission.columns.equals(
+        new_test.columns
+    ), "Column mismatch between sample_submission and private test"
+    assert new_train.columns.equals(
+        original_meta_subset.columns
+    ), f"Unexpected columns in train, expected {original_meta_subset.columns}, got {new_train.columns}"
+    assert new_test_without_labels.columns.equals(
+        original_meta_subset.drop(columns=["azimuth", "zenith"]).columns
+    ), f"Unexpected columns in test"
+
+    assert (
+        len(set(new_train["event_id"]).intersection(set(new_test["event_id"]))) == 0
+    ), "Event ids overlap between train and test"
+    assert set(new_test["event_id"]) == set(
+        sample_submission["event_id"]
+    ), "Event ids mismatch between test and sample submission"
+
+
+def prepare(raw: Path, public: Path, private: Path):
+    DEV = False
+
+    if DEV:
+        batch_cutoff = 66  # 66 instead of 660 when in dev mode
+    else:
+        batch_cutoff = None
+
+    logger.info("Loading raw metadata")
+    old_train = pd.read_parquet(raw / "train_meta.parquet")
+    batch_ids = old_train["batch_id"].unique()[:batch_cutoff]
+
+    # Clean the dataframe for processing.
+    # The original script adds a 'split' column which we don't need in the helper.
+    old_train_clean = old_train[old_train["batch_id"].isin(batch_ids)].drop(
+        columns=["split"], errors="ignore"
+    )
+
+    # --- 1. Original Split: Raw Data -> Train / Test ---
+    logger.info("Splitting batches into original train and test sets")
+    train_batch_ids, test_batch_ids = train_test_split(
+        batch_ids, test_size=0.1, random_state=0
+    )
+
+    # Process and save the original split to 'public' and 'private'
+    _create_split_files(
+        original_meta_df=old_train_clean,
+        train_batch_ids=list(train_batch_ids),
+        test_batch_ids=list(test_batch_ids),
+        public_dir=public,
+        private_dir=private,
+        raw_dir=raw,
+    )
+
+    # --- 2. New Validation Split: Original Train -> New Train / Validation Test ---
+    logger.info("Creating new parallel directories for validation split")
+    public_val = public.parent / "public_val"
+    private_val = private.parent / "private_val"
+
+    # The new validation set should be the same size as the original test set.
+    # Calculate the required test_size for the second split.
+    # test_size = (size of desired val set) / (size of set being split)
+    #           = len(test_batch_ids) / len(train_batch_ids)
+    test_val_size_ratio = len(test_batch_ids) / len(train_batch_ids)
+
+    logger.info(f"Splitting original train set to create validation set")
+    train_val_batch_ids, test_val_batch_ids = train_test_split(
+        train_batch_ids, test_size=test_val_size_ratio, random_state=0
+    )
+
+    # Process and save the new validation split to 'public_val' and 'private_val'
+    _create_split_files(
+        original_meta_df=old_train_clean,
+        train_batch_ids=list(train_val_batch_ids),
+        test_batch_ids=list(test_val_batch_ids),
+        public_dir=public_val,
+        private_dir=private_val,
+        raw_dir=raw,
+    )
+
+    logger.info("All data preparation tasks are complete.")

mlebench/competitions/ili/grade.py

@@ -0,0 +1,60 @@
+import pandas as pd
+import numpy as np
+
+
+def grade(submission: pd.DataFrame, answers: pd.DataFrame) -> float:
+    """
+    Grade the submission using MSE metric.
+
+    Args:
+        submission: DataFrame with columns ['id', 'pred_0', 'pred_1', ..., 'pred_167']
+                   Each row represents one test sample
+                   Columns pred_0 to pred_167 represent flattened predictions (24 timesteps * 7 features)
+        answers: DataFrame with columns ['id', 'pred_0', 'pred_1', ..., 'pred_167']
+                Ground truth in same format as submission
+
+    Returns:
+        MSE score (float) - lower is better
+    """
+    try:
+        # Extract prediction columns (pred_0 to pred_167 for 24*7=168 values)
+        pred_cols = [f'pred_{i}' for i in range(24 * 7)]
+
+        # Check if all required columns exist
+        if not all(col in submission.columns for col in pred_cols):
+            print(f"Error: Missing prediction columns. Expected columns: {pred_cols[:5]}...{pred_cols[-5:]}")
+            return float(1e10)
+
+        # Sort by id to ensure alignment
+        submission = submission.sort_values('id').reset_index(drop=True)
+        answers = answers.sort_values('id').reset_index(drop=True)
+
+        # Extract predictions
+        predictions = submission[pred_cols].values
+        ground_truth = answers[pred_cols].values
+
+        # Reshape to (N, 24, 7) for MSE calculation
+        predictions_3d = predictions.reshape(-1, 24, 7)
+        ground_truth_3d = ground_truth.reshape(-1, 24, 7)
+
+        # Ensure shapes match
+        if predictions_3d.shape != ground_truth_3d.shape:
+            print(f"Error: Shape mismatch. Got {predictions_3d.shape}, expected {ground_truth_3d.shape}")
+            return float(1e10)
+
+        # Calculate MSE (primary metric)
+        mse = np.mean((predictions_3d - ground_truth_3d) ** 2)
+
+        # Calculate MAE (for logging/debugging)
+        mae = np.mean(np.abs(predictions_3d - ground_truth_3d))
+
+        print(f"MSE: {mse:.6f}, MAE: {mae:.6f}")
+
+        return float(mse)
+
+    except Exception as e:
+        print(f"Error during grading: {e}")
+        import traceback
+        traceback.print_exc()
+        # Return a very high score (bad) on error
+        return float(1e10)

mlebench/competitions/ili/prepare.py

@@ -0,0 +1,99 @@
+import pandas as pd
+import numpy as np
+import sys
+from pathlib import Path
+
+
+def prepare(raw: Path, public: Path, private: Path):
+    """
+    Prepare the ILI dataset for the benchmark.
+
+    This function converts the CSV time series data to numpy arrays for
+    time series forecasting.
+
+    Args:
+        raw: Path to raw data directory (contains national_illness.csv)
+        public: Path to public directory (visible to participants)
+        private: Path to private directory (hidden from participants, used for grading)
+    """
+    try:
+        # Add raw directory to path temporarily to import dataset module
+        sys.path.insert(0, str(raw))
+
+        # Use the local dataset module to load data
+        from dataset import get_dataset
+
+        # Change to raw directory to load data
+        import os
+        original_dir = os.getcwd()
+        os.chdir(str(raw))
+
+        try:
+            # Load train and test datasets
+            X_train, y_train = get_dataset('train')
+            X_test, y_test = get_dataset('test')
+
+            print(f"Loaded training data: X_train.shape = {X_train.shape}, y_train.shape = {y_train.shape}")
+            print(f"Loaded test data: X_test.shape = {X_test.shape}, y_test.shape = {y_test.shape}")
+
+            # Save training data as numpy arrays in public directory
+            np.save(public / "train_input.npy", X_train)
+            np.save(public / "train_target.npy", y_train)
+
+            # Save test input (without targets) in public directory
+            np.save(public / "test_input.npy", X_test)
+
+            # Save test targets for grading (private)
+            # Save as both numpy (for convenience) and CSV (for framework compatibility)
+            np.save(private / "test_targets.npy", y_test)
+
+            # Also save as CSV for framework compatibility
+            test_targets_flat = y_test.reshape(len(y_test), -1)
+            test_targets_df = pd.DataFrame(test_targets_flat,
+                                          columns=[f'pred_{i}' for i in range(24 * 7)])
+            test_targets_df.insert(0, 'id', range(len(y_test)))
+            test_targets_df.to_csv(private / "test_targets.csv", index=False)
+
+            # Create sample submission CSV file
+            # Flatten (N, 24, 7) to (N, 168) where 168 = 24*7
+            sample_submission_flat = np.zeros((len(y_test), 24 * 7))
+            sample_submission_df = pd.DataFrame(sample_submission_flat,
+                                               columns=[f'pred_{i}' for i in range(24 * 7)])
+            sample_submission_df.insert(0, 'id', range(len(y_test)))
+            sample_submission_df.to_csv(public / "sample_submission.csv", index=False)
+
+            # Also save numpy version for convenience
+            np.save(public / "sample_submission.npy", np.zeros_like(y_test))
+
+            print(f"Data preparation completed:")
+            print(f"  - Training: {len(X_train)} samples")
+            print(f"  - Test: {len(X_test)} samples")
+            print(f"  - Input sequence length: {X_train.shape[1]}")
+            print(f"  - Input features: {X_train.shape[2]}")
+            print(f"  - Output sequence length: {y_train.shape[1]}")
+            print(f"  - Output features: {y_train.shape[2]}")
+
+        finally:
+            os.chdir(original_dir)
+            sys.path.remove(str(raw))
+
+    except Exception as e:
+        print(f"Error loading data: {e}")
+        import traceback
+        traceback.print_exc()
+        # Fallback: create dummy files if loading fails
+        print("Creating dummy submission files...")
+        dummy_array = np.zeros((1, 24, 7))
+        np.save(private / "test_targets.npy", dummy_array)
+        np.save(public / "sample_submission.npy", dummy_array)
+
+    # Validation checks
+    assert (public / "train_input.npy").exists(), "Training input should exist"
+    assert (public / "train_target.npy").exists(), "Training target should exist"
+    assert (public / "test_input.npy").exists(), "Test input should exist"
+    assert (private / "test_targets.npy").exists(), "Test targets should exist"
+    assert (public / "sample_submission.npy").exists(), "Sample submission should exist"
+
+    print(f"\nPrepared ILI dataset:")
+    print(f"  - Public files: {list(public.glob('*'))}")
+    print(f"  - Private files: {list(private.glob('*'))}")

mlebench/competitions/imet-2020-fgvc7/grade.py

@@ -0,0 +1,54 @@
+import pandas as pd
+from scipy.sparse import csr_matrix
+from sklearn.metrics import f1_score
+from sklearn.preprocessing import MultiLabelBinarizer
+
+from mlebench.grade_helpers import InvalidSubmissionError
+
+
+def prepare_for_metric(
+    submission: pd.DataFrame, answers: pd.DataFrame
+) -> tuple[csr_matrix, csr_matrix]:
+    """Transforms the submission and answers DataFrames into the required format for grading as sparse matrices."""
+
+    # Answers checks
+    assert (
+        "id" in answers.columns and "attribute_ids" in answers.columns
+    ), "Answers DataFrame must have 'id' and 'attribute_ids' columns"
+
+    # Submission checks
+    if "id" not in submission.columns:
+        raise InvalidSubmissionError("Submission DataFrame must have an 'id' column.")
+    if "attribute_ids" not in submission.columns:
+        raise InvalidSubmissionError("Submission DataFrame must have a 'attribute_ids' column.")
+    if len(submission) != len(answers):
+        raise InvalidSubmissionError(
+            "Submission and answers DataFrames must have the same number of rows."
+        )
+
+    # Match order
+    submission = submission.sort_values(by="id").reset_index(drop=True)
+    answers = answers.sort_values(by="id").reset_index(drop=True)
+    if (submission["id"].values != answers["id"].values).any():
+        raise InvalidSubmissionError(
+            "Submission and answers DataFrames must have matching 'id' columns."
+        )
+
+    # pandas reads empty cells as NaNs, which are float. We fill with empty string to match type
+    submission["attribute_ids"] = submission["attribute_ids"].fillna("")
+    answers["attribute_ids"] = answers["attribute_ids"].fillna("")
+
+    # Get classes
+    classes = set(answers["attribute_ids"].str.split().explode().unique())
+
+    # Convert to sparse matrices using MultiLabelBinarizer
+    mlb = MultiLabelBinarizer(classes=sorted(classes), sparse_output=True)
+    y_true = mlb.fit_transform(answers["attribute_ids"].str.split())
+    y_pred = mlb.transform(submission["attribute_ids"].str.split())
+
+    return y_true, y_pred
+
+
+def grade(submission: pd.DataFrame, answers: pd.DataFrame) -> float:
+    y_true, y_pred = prepare_for_metric(submission, answers)
+    return f1_score(y_true=y_true, y_pred=y_pred, average="micro")

mlebench/competitions/imet-2020-fgvc7/prepare.py

@@ -0,0 +1,77 @@
+import shutil
+from pathlib import Path
+
+from sklearn.model_selection import train_test_split
+from tqdm import tqdm
+
+from mlebench.utils import read_csv
+
+
+def prepare(raw: Path, public: Path, private: Path):
+
+    dev = False
+
+    # Create train, test from train split
+    old_train = read_csv(raw / "train.csv")
+    # 25958/(142119+ 25958) = 0.15 original test split
+    new_train, answers = train_test_split(old_train, test_size=0.15, random_state=0)
+
+    # Sample submission
+    sample_submission_df = answers.copy()
+    sample_submission_df["attribute_ids"] = "0 1 2"
+
+    # Checks
+    assert len(answers) + len(new_train) == len(
+        old_train
+    ), "New train and test should have the same length as the old train"
+    assert len(answers) == len(
+        sample_submission_df
+    ), "Answers and sample submission should have the same length"
+    assert (
+        answers.columns.tolist() == old_train.columns.tolist()
+    ), "Answers and old_train should have the same columns"
+    assert (
+        sample_submission_df.columns.tolist() == old_train.columns.tolist()
+    ), "Sample submission and old train should have the same columns"
+    assert (
+        new_train.columns.tolist() == old_train.columns.tolist()
+    ), "New train and old train should have the same columns"
+    assert set(new_train["id"]).isdisjoint(
+        set(answers["id"])
+    ), "New train and answers should have no overlap of rows"
+
+    # Write CSVs
+    answers.to_csv(private / "answers.csv", index=False)
+    new_train.to_csv(public / "train.csv", index=False)
+    sample_submission_df.to_csv(public / "sample_submission.csv", index=False)
+
+    # Copy over files
+    shutil.copyfile(src=raw / "labels.csv", dst=public / "labels.csv")
+
+    (public / "test").mkdir(exist_ok=True)
+    (public / "train").mkdir(exist_ok=True)
+
+    if dev:
+        # Reduce the number of images to be copied to 10% of the original, for testing
+        new_train = new_train.head(int(len(new_train) * 0.1))
+        answers = answers.head(int(len(answers) * 0.1))
+
+    for file_id in tqdm(new_train["id"], desc="Copying train images"):
+        shutil.copyfile(
+            src=raw / "train" / f"{file_id}.png",
+            dst=public / "train" / f"{file_id}.png",
+        )
+
+    for file_id in tqdm(answers["id"], desc="Copying test images"):
+        shutil.copyfile(
+            src=raw / "train" / f"{file_id}.png",
+            dst=public / "test" / f"{file_id}.png",
+        )
+
+    # File copying checks
+    assert len(list(public.glob("train/*.png"))) == len(
+        new_train
+    ), "Train images should have the same number of images as the train"
+    assert len(list(public.glob("test/*.png"))) == len(
+        answers
+    ), "Test images should have the same number of images as the test"

mlebench/competitions/imet-2020-fgvc7/prepare_val.py

@@ -0,0 +1,157 @@
+import shutil
+from pathlib import Path
+
+from sklearn.model_selection import train_test_split
+from tqdm import tqdm
+
+from mlebench.utils import read_csv
+
+
+def _process_split(
+    train_df,
+    test_df,
+    public_path: Path,
+    private_path: Path,
+    raw_images_path: Path,
+    dev: bool,
+):
+    """
+    Helper function to process a single data split (train/test).
+
+    It handles the creation of directories, writing of CSVs (train, answers,
+    sample_submission), and copying of image files. This ensures a consistent
+    output structure for any given split.
+    """
+    # Create output directories
+    public_path.mkdir(exist_ok=True, parents=True)
+    private_path.mkdir(exist_ok=True, parents=True)
+    (public_path / "test").mkdir(exist_ok=True)
+    (public_path / "train").mkdir(exist_ok=True)
+
+    # The test_df is the ground truth for the test set
+    answers = test_df
+
+    # Create a sample submission from the answers dataframe
+    sample_submission_df = answers.copy()
+    sample_submission_df["attribute_ids"] = "0 1 2"
+
+    # Checks
+    assert len(answers) == len(
+        sample_submission_df
+    ), "Answers and sample submission should have the same length"
+    assert (
+        sample_submission_df.columns.tolist() == answers.columns.tolist()
+    ), "Sample submission and answers should have the same columns"
+
+    # Write CSVs
+    train_df.to_csv(public_path / "train.csv", index=False)
+    answers.to_csv(private_path / "answers.csv", index=False)
+    sample_submission_df.to_csv(public_path / "sample_submission.csv", index=False)
+
+    # If in dev mode, reduce the number of images to copy for faster execution
+    if dev:
+        train_df_to_copy = train_df.head(int(len(train_df) * 0.1))
+        answers_to_copy = answers.head(int(len(answers) * 0.1))
+    else:
+        train_df_to_copy = train_df
+        answers_to_copy = answers
+
+    # Copy train images
+    for file_id in tqdm(
+        train_df_to_copy["id"], desc=f"Copying train images to {public_path.name}"
+    ):
+        shutil.copyfile(
+            src=raw_images_path / f"{file_id}.png",
+            dst=public_path / "train" / f"{file_id}.png",
+        )
+
+    # Copy test images
+    for file_id in tqdm(
+        answers_to_copy["id"], desc=f"Copying test images to {public_path.name}"
+    ):
+        shutil.copyfile(
+            src=raw_images_path / f"{file_id}.png",
+            dst=public_path / "test" / f"{file_id}.png",
+        )
+
+    # File copying checks
+    assert len(list(public_path.glob("train/*.png"))) == len(
+        train_df_to_copy
+    ), f"Train images in {public_path.name} should match dataframe"
+    assert len(list(public_path.glob("test/*.png"))) == len(
+        answers_to_copy
+    ), f"Test images in {public_path.name} should match dataframe"
+
+
+def prepare(raw: Path, public: Path, private: Path):
+
+    dev = False
+
+    # --- 1. Original Split: Create main train and test sets ---
+
+    # Create train, test from train split
+    old_train = read_csv(raw / "train.csv")
+    # 25958/(142119+ 25958) = 0.15 original test split
+    main_train, main_test = train_test_split(old_train, test_size=0.15, random_state=0)
+
+    # Checks
+    assert len(main_test) + len(main_train) == len(
+        old_train
+    ), "New train and test should have the same length as the old train"
+    assert (
+        main_test.columns.tolist() == old_train.columns.tolist()
+    ), "Answers and old_train should have the same columns"
+    assert (
+        main_train.columns.tolist() == old_train.columns.tolist()
+    ), "New train and old train should have the same columns"
+    assert set(main_train["id"]).isdisjoint(
+        set(main_test["id"])
+    ), "New train and answers should have no overlap of rows"
+
+    # Process and save the original split to `public` and `private` directories
+    _process_split(
+        train_df=main_train,
+        test_df=main_test,
+        public_path=public,
+        private_path=private,
+        raw_images_path=raw / "train",
+        dev=dev,
+    )
+
+    # Copy over metadata file for the original split
+    shutil.copyfile(src=raw / "labels.csv", dst=public / "labels.csv")
+
+    # --- 2. New Validation Split: Create validation train and test sets ---
+
+    # Define paths for the new validation split, parallel to the original ones
+    public_val = public.parent / "public_val"
+    private_val = private.parent / "private_val"
+
+    # The test_size for the second split is calculated to yield a validation set
+    # of approximately the same size as the original test set.
+    # Original test set size = 0.15 * N
+    # Original train set size = 0.85 * N
+    # We want: test_size_val * (0.85 * N) ~= 0.15 * N
+    # So, test_size_val ~= 0.15 / 0.85
+    test_size_for_val_split = 0.15 / (1 - 0.15)
+
+    # Split the main_train set again to create a smaller training set and a validation set.
+    # We use the same random_state to ensure this split is also deterministic.
+    train_val, test_val = train_test_split(
+        main_train, test_size=test_size_for_val_split, random_state=0
+    )
+
+    # Process and save the validation split to `public_val` and `private_val`.
+    # The filenames and structure will be identical to the original split, but
+    # will contain the new, smaller training set and the validation set.
+    _process_split(
+        train_df=train_val,
+        test_df=test_val,
+        public_path=public_val,
+        private_path=private_val,
+        raw_images_path=raw / "train",
+        dev=dev,
+    )
+
+    # Copy over metadata file for the validation split
+    shutil.copyfile(src=raw / "labels.csv", dst=public_val / "labels.csv")

mlebench/competitions/inaturalist-2019-fgvc6/grade.py

@@ -0,0 +1,35 @@
+import pandas as pd
+from sklearn.metrics import accuracy_score
+
+from mlebench.grade_helpers import InvalidSubmissionError
+
+
+def prepare_for_metric(submission: pd.DataFrame, answers: pd.DataFrame):
+    """
+    `submission` and `answers` are pd.DataFrame with "id" and "predicted" columns
+    """
+    if len(submission) != len(answers):
+        raise InvalidSubmissionError(
+            f"Submission length {len(submission)} != answers length {len(answers)}"
+        )
+    if "id" not in submission.columns or "predicted" not in submission.columns:
+        raise InvalidSubmissionError("Submission must have 'id' and 'predicted' columns")
+
+    assert "id" in answers.columns, "Answers must have 'id' column"
+    assert "predicted" in answers.columns, "Answers must have 'predicted' column"
+
+    # Sort submission and answers by "id"
+    submission = submission.sort_values("id")
+    answers = answers.sort_values("id")
+    if (submission["id"].values != answers["id"].values).any():
+        raise InvalidSubmissionError("Submission and answers have mismatched 'id' columns")
+
+    y_true = [int(y) for y in answers["predicted"]]
+    y_pred = [int(y) if isinstance(y, int) else int(y.split()[0]) for y in submission["predicted"]]
+    return y_true, y_pred
+
+
+def grade(submission: pd.DataFrame, answers: pd.DataFrame) -> float:
+    y_true, y_pred = prepare_for_metric(submission, answers)
+    accuracy = accuracy_score(y_true=y_true, y_pred=y_pred)
+    return 1 - accuracy