dslighting 1.3.9__py3-none-any.whl → 1.7.1__py3-none-any.whl

dslighting/registry/tgs-salt-identification-challenge/prepare_val.py

@@ -0,0 +1,166 @@
+import shutil
+from pathlib import Path
+
+import pandas as pd
+from sklearn.model_selection import train_test_split
+
+from mlebench.utils import extract, read_csv
+
+
+def _process_split(
+    train_df: pd.DataFrame,
+    test_df: pd.DataFrame,
+    public_dir: Path,
+    private_dir: Path,
+    raw_images_dir: Path,
+    raw_masks_dir: Path,
+    all_depths_df: pd.DataFrame,
+) -> None:
+    """
+    Processes a single data split (e.g., train/test or train_val/test_val),
+    saving all required files and performing sanity checks.
+
+    Args:
+        train_df: DataFrame for the training set.
+        test_df: DataFrame for the test set.
+        public_dir: The public output directory for this split.
+        private_dir: The private output directory for this split.
+        raw_images_dir: Path to the directory containing all source images.
+        raw_masks_dir: Path to the directory containing all source masks.
+        all_depths_df: DataFrame containing depth information for all samples.
+    """
+    public_dir.mkdir(parents=True, exist_ok=True)
+    private_dir.mkdir(parents=True, exist_ok=True)
+
+    train_ids = set(train_df["id"])
+    test_ids = set(test_df["id"])
+
+    assert train_ids.isdisjoint(test_ids), "`id` is not disjoint between train and test sets"
+
+    train_df.sort_values(by="id").to_csv(public_dir / "train.csv", index=False)
+    test_df.sort_values(by="id").to_csv(private_dir / "test.csv", index=False)
+
+    all_source_images = set(raw_images_dir.glob("*.png"))
+    train_imgs = set(img for img in all_source_images if img.stem in train_ids)
+    test_imgs = set(img for img in all_source_images if img.stem in test_ids)
+
+    assert train_imgs.isdisjoint(test_imgs), "Images are not disjoint between train and test sets"
+
+    (public_dir / "train" / "images").mkdir(parents=True, exist_ok=True)
+    (public_dir / "train" / "masks").mkdir(parents=True, exist_ok=True)
+
+    for fpath in train_imgs:
+        shutil.copyfile(src=fpath, dst=public_dir / "train" / "images" / fpath.name)
+        shutil.copyfile(src=raw_masks_dir / fpath.name, dst=public_dir / "train" / "masks" / fpath.name)
+
+    (public_dir / "test" / "images").mkdir(parents=True, exist_ok=True)
+
+    for fpath in test_imgs:
+        shutil.copyfile(src=fpath, dst=public_dir / "test" / "images" / fpath.name)
+
+    sample_submission = test_df.drop(columns=["rle_mask"]).copy()
+    sample_submission["rle_mask"] = "1 1"
+    sample_submission.sort_values(by="id").to_csv(public_dir / "sample_submission.csv", index=False)
+
+    depths_mask = all_depths_df["id"].isin(train_ids)
+    filtered_depths = all_depths_df[depths_mask]
+    filtered_depths.sort_values(by="id").to_csv(public_dir / "depths.csv", index=False)
+
+    # Sanity checks
+    assert (public_dir / "train.csv").exists(), "`train.csv` doesn't exist!"
+    assert (public_dir / "sample_submission.csv").exists(), "`sample_submission.csv` doesn't exist!"
+    assert (public_dir / "depths.csv").exists(), "`depths.csv` doesn't exist!"
+    assert (public_dir / "train").exists(), "`train` directory doesn't exist!"
+    assert (public_dir / "test").exists(), "`test` directory doesn't exist!"
+    assert (private_dir / "test.csv").exists(), "`test.csv` doesn't exist!"
+
+    actual_train_imgs = set(img.stem for img in (public_dir / "train" / "images").glob("*.png"))
+    actual_train_masks = set(img.stem for img in (public_dir / "train" / "masks").glob("*.png"))
+
+    assert len(actual_train_imgs) == len(train_df), "The number of images in the train set doesn't match!"
+    assert len(actual_train_masks) == len(train_df), "The number of masks in the train set doesn't match!"
+
+    for train_id in train_df["id"]:
+        assert (public_dir / "train" / "images" / f"{train_id}.png").exists()
+        assert (public_dir / "train" / "masks" / f"{train_id}.png").exists()
+
+    actual_test_imgs = set(img.stem for img in (public_dir / "test" / "images").glob("*.png"))
+
+    assert not (public_dir / "test" / "masks").exists(), f"Expected `{public_dir}/test/masks` to not exist, but it does!"
+    assert len(actual_test_imgs) == len(test_df), "The number of images in the test set doesn't match!"
+
+    for test_id in test_df["id"]:
+        assert (public_dir / "test" / "images" / f"{test_id}.png").exists()
+        assert not (public_dir / "test" / "masks" / f"{test_id}.png").exists()
+
+    assert actual_train_imgs.isdisjoint(actual_test_imgs), "Image sets overlap!"
+
+    actual_sample_submission = read_csv(public_dir / "sample_submission.csv")
+    actual_test = read_csv(private_dir / "test.csv")
+
+    assert len(actual_sample_submission) == len(actual_test), "Sample submission and test set lengths differ!"
+    assert set(actual_sample_submission["id"]) == set(actual_test["id"]), "Sample submission and test set IDs differ!"
+    assert len(actual_test_imgs) == len(actual_test), "Test image count and test set length differ!"
+    assert set(actual_test["id"]) == actual_test_imgs, "Test set IDs and test images differ!"
+
+
+def prepare(raw: Path, public: Path, private: Path) -> None:
+    extract(raw / "competition_data.zip", raw)
+
+    old_train = read_csv(raw / "competition_data" / "train.csv")
+    old_train = old_train.fillna("")
+    old_depths = read_csv(raw / "depths.csv")
+
+    # Original ratio is Train set - 4,000 samples; Test set - ~18,000 samples (82% ratio)
+    # We use a 0.25 ratio to get number of test samples into thousand OOM
+    new_train, new_test = train_test_split(old_train, test_size=0.25, random_state=0)
+
+    assert len(new_train) + len(new_test) == len(
+        old_train
+    ), "Some samples were lost when creating the new train and test sets!"
+
+    # Create the new validation split from the `new_train` set.
+    # To make `test_val` have the same size as `new_test` (25% of original),
+    # we need to take 1/3 of `new_train` (since 1/3 * 75% = 25%).
+    train_val, test_val = train_test_split(new_train, test_size=(1/3), random_state=0)
+
+    assert len(train_val) + len(test_val) == len(
+        new_train
+    ), "Some samples were lost when creating the validation train and test sets!"
+
+    # Define paths for raw images and new validation output directories
+    raw_images_dir = raw / "competition_data" / "train" / "images"
+    raw_masks_dir = raw / "competition_data" / "train" / "masks"
+    public_val = public.parent / "public_val"
+    private_val = private.parent / "private_val"
+
+    # Process the original split, saving to `public` and `private`
+    # This block ensures the original outputs are not modified.
+    _process_split(
+        train_df=new_train,
+        test_df=new_test,
+        public_dir=public,
+        private_dir=private,
+        raw_images_dir=raw_images_dir,
+        raw_masks_dir=raw_masks_dir,
+        all_depths_df=old_depths,
+    )
+
+    # Process the new validation split, saving to `public_val` and `private_val`
+    _process_split(
+        train_df=train_val,
+        test_df=test_val,
+        public_dir=public_val,
+        private_dir=private_val,
+        raw_images_dir=raw_images_dir,
+        raw_masks_dir=raw_masks_dir,
+        all_depths_df=old_depths,
+    )
+
+    # Final checks on data types, which are consistent across all splits.
+    assert new_train.applymap(
+        lambda x: isinstance(x, str)
+    ).values.all(), "Not all elements in the DataFrame are strings!"
+    assert new_test.applymap(
+        lambda x: isinstance(x, str)
+    ).values.all(), "Not all elements in the DataFrame are strings!"

dslighting/registry/the-icml-2013-whale-challenge-right-whale-redux/checksums.yaml

@@ -0,0 +1,5 @@
+private:
+  test.csv: 199fdb3b74d7e1bc30fabc4fe4c66743
+public:
+  sampleSubmission.csv: 76160a892fbd2db6d847bd444f576cee
+zip: 31d4fe71434eaa84fb0003374e6a30cc

dslighting/registry/the-icml-2013-whale-challenge-right-whale-redux/config.yaml

@@ -0,0 +1,20 @@
+id: the-icml-2013-whale-challenge-right-whale-redux
+name: The ICML 2013 Whale Challenge - Right Whale Redux
+competition_type: simple
+awards_medals: true
+prizes:
+  - position: 1
+    value: 350
+  - position: 2
+    value: 150
+description: mlebench/competitions/the-icml-2013-whale-challenge-right-whale-redux/description.md
+
+dataset:
+  answers: the-icml-2013-whale-challenge-right-whale-redux/prepared/private/test.csv
+  sample_submission: the-icml-2013-whale-challenge-right-whale-redux/prepared/public/sampleSubmission.csv
+
+grader:
+  name: auc-roc
+  grade_fn: mlebench.competitions.the-icml-2013-whale-challenge-right-whale-redux.grade:grade
+
+preparer: mlebench.competitions.the-icml-2013-whale-challenge-right-whale-redux.prepare:prepare

dslighting/registry/the-icml-2013-whale-challenge-right-whale-redux/description.md

@@ -0,0 +1,72 @@
+# Overview
+
+## Description
+
+![Right whale](https://storage.googleapis.com/kaggle-media/competitions/kaggle/3509/media/right_whale.jpg)*(right whale illustration courtesy of Pieter Folkens, ©2011)*
+
+This competition complements the previously held [Marinexplore Whale Detection Challenge](https://www.kaggle.com/c/whale-detection-challenge), in which Cornell University provided data from a ship monitoring application termed "Auto Buoy", or AB Monitoring System. In the Marinexplore challenge we received solutions that exceeded 98% accuracy and will ultimately advance the process of automatically classifying North Atlantic Right Whales using the AB Monitoring Platform.
+
+Since the results from the previous challenge proved so successful, we decided to extend the goals and consider applications that involve running algorithms on archival data recorded using portable hydrophone assemblies, otherwise referred to as Marine Autonomous Recording Unit (or MARU's). Since Cornell and its partners have been using the MARU for over a decade, a sizable collection of data has been accumulated. This data spans several ocean basins and covers a variety of marine mammal species.
+
+Solutions to this challenge will be ported to a High Performance Computing (HPC) platform, being developed in part through funding provided by the Office of Naval Research (ONR grant N000141210585, Dugan, Clark, LeCun and Van Parijs). Together, Cornell will combine algorithms, HPC technologies and its data archives to explore data using highly accurate measuring tools. We encourage participants who developed prior solutions (through the collaboration with Marinexplore) to test them on this data.
+
+The results will be presented at the [Workshop on Machine Learning for Bioacoustics](http://sabiod.univ-tln.fr/icml2013/challenge_description.html) at ICML 2013.
+
+## Prizes
+
+-   First place - $350
+-   Second place - $150
+
+## Evaluation
+
+Submissions are judged on [area under the ROC curve](http://en.wikipedia.org/wiki/Receiver_operating_characteristic). 
+
+In Matlab (using the stats toolbox):
+
+```
+[~, ~, ~, auc ] = perfcurve(true_labels, predictions, 1);
+```
+
+In R (using the verification package):
+
+```
+auc = roc.area(true_labels, predictions)
+```
+
+In python (using the metrics module of scikit-learn):
+
+```
+fpr, tpr, thresholds = metrics.roc_curve(true_labels, predictions, pos_label=1)
+auc = metrics.auc(fpr,tpr)
+```
+
+## Submission File
+
+For every recording in the test set, submission files should contain two columns: clip and probability. The submission file should have a header.
+
+```
+clip,probability
+20090404_000000_012s0ms_Test0.aif,0
+20090404_000000_042s6ms_Test1.aif,0
+20090404_000000_064s8ms_Test2.aif,0
+20090404_000000_082s3ms_Test3.aif,0
+20090404_000000_095s0ms_Test4.aif,0
+```
+
+### Citation
+
+DeLMA, Will Cukierski. (2013). The ICML 2013 Whale Challenge - Right Whale Redux. Kaggle. https://kaggle.com/competitions/the-icml-2013-whale-challenge-right-whale-redux
+
+
+# Data
+
+## Dataset Description
+
+You are given four days of training data and three days of testing data.  The task is to assign a probability that each recording in the test set contains a right whale call (1) or noise (0).
+
+### File descriptions
+
+- **train2.zip** - all of the training data as .aif clips. If the file ends in "_1.aif" it was labeled a whale call, if it ends in "_0.aif", it was labeled noise.
+- **test2.zip** -  all of the testing data as .aif clips. You should predict a probability for each of these files.
+
+To run within hardware constraints, solutions would ideally function with limited training history, meaning the algorithm would page in no more than five previous minutes of data at a time. Since we cannot enforce this requirement in this particular competition, we provide this info for those who want to optimize on "real world utility," as opposed to AUC.

dslighting/registry/the-icml-2013-whale-challenge-right-whale-redux/description_obfuscated.md

@@ -0,0 +1,25 @@
+# Task
+
+Given a recording, predict if it contains a right whale call (1) or noise (0).
+
+# Metric
+
+Area under the ROC curve.
+
+# Submission Format
+
+For every recording in the test set, submission files should contain two columns: clip and probability. The submission file should have a header.
+
+```
+clip,probability
+20090404_000000_012s0ms_Test0.aif,0
+20090404_000000_042s6ms_Test1.aif,0
+20090404_000000_064s8ms_Test2.aif,0
+20090404_000000_082s3ms_Test3.aif,0
+20090404_000000_095s0ms_Test4.aif,0
+```
+
+# Dataset 
+
+- **train2.zip** - all of the training data as .aif clips. If the file ends in "_1.aif" it was labeled a whale call, if it ends in "_0.aif", it was labeled noise.
+- **test2.zip** -  all of the testing data as .aif clips. You should predict a probability for each of these files.

dslighting/registry/the-icml-2013-whale-challenge-right-whale-redux/grade.py

@@ -0,0 +1,11 @@
+import pandas as pd
+from sklearn.metrics import roc_auc_score
+
+from mlebench.competitions.utils import prepare_for_auroc_metric
+
+
+def grade(submission: pd.DataFrame, answers: pd.DataFrame) -> float:
+    roc_auc_inputs = prepare_for_auroc_metric(
+        submission, answers, id_col="clip", target_col="probability"
+    )
+    return roc_auc_score(y_true=roc_auc_inputs["y_true"], y_score=roc_auc_inputs["y_score"])

dslighting/registry/the-icml-2013-whale-challenge-right-whale-redux/prepare.py

@@ -0,0 +1,95 @@
+import re
+import shutil
+from pathlib import Path
+
+import pandas as pd
+from tqdm import tqdm
+
+
+def prepare(raw: Path, public: Path, private: Path):
+    """
+    Splits the data in raw into public and private datasets with appropriate test/train splits.
+    """
+    # Data is in train2.zip - we need to unzip it
+    shutil.unpack_archive(raw / "train2.zip", raw)
+
+    # Files are named as
+    # Train: "YYYYMMDD_HHMMSS_{seconds}_TRAIN{idx}_{label:0,1}.aif"
+    # Test: "YYYYMMDD_HHMMSS_{seconds}_Test{idx}.aif"
+
+    # There are 4 days in Train and 3 days in Test
+    # In our new dataset, we'll just split Train_old into 2 days for Train and 2 days for Test
+
+    samples_by_date = {}
+    n_train_old = 0
+    for sample in (raw / "train2").iterdir():
+        date = sample.name.split("_")[0]
+        if date not in samples_by_date:
+            samples_by_date[date] = []
+        samples_by_date[date].append(sample)
+        n_train_old += 1
+
+    assert len(samples_by_date) == 4, "Expected 4 days in Train_old"
+    dates = sorted(list(samples_by_date.keys()))
+    new_train = samples_by_date[dates[0]] + samples_by_date[dates[1]]
+    new_test = samples_by_date[dates[2]] + samples_by_date[dates[3]]
+    # Sort files - filenames have timestamps so we want new idxs to be time-ordered
+    new_train = sorted(new_train)
+    new_test = sorted(new_test)
+
+    # Copy files to new directories
+    (public / "train2").mkdir(exist_ok=True, parents=True)
+    for idx, sample in enumerate(tqdm(new_train)):
+        # Replace index part of filename with new index
+        new_sample_name = re.sub(r"TRAIN\d+", f"TRAIN{idx}", sample.name)
+        new_sample = public / "train2" / new_sample_name
+        shutil.copy(sample, new_sample)
+
+    answer_rows = []  # While we're at it, collect answers for the new test set
+    (public / "test2").mkdir(exist_ok=True, parents=True)
+    for idx, sample in enumerate(tqdm(new_test)):
+        # Replace everything after the TRAIN{idx} part of the filename
+        # (replaces index as well as label part of filename)
+        new_sample_name = sample.name.split("TRAIN")[0] + f"Test{idx}.aif"
+        new_sample = public / "test2" / new_sample_name
+        shutil.copy(sample, new_sample)
+
+        # Add to new test set answers
+        answer_rows.append(
+            {"clip": new_sample_name, "probability": 1 if sample.stem.endswith("_1") else 0}
+        )
+
+    assert len(new_train) == len(
+        list((public / "train2").glob("*.aif"))
+    ), f"Expected {len(new_train)} samples in new_train ({len(list((public / 'train2').glob('*.aif')))}"
+    assert len(new_test) == len(
+        list((public / "test2").glob("*.aif"))
+    ), f"Expected {len(new_test)} samples in new_test ({len(list((public / 'test2').glob('*.aif')))}"
+    assert (
+        len(new_train) + len(new_test) == n_train_old
+    ), f"Expected {n_train_old} total samples in new_train ({len(new_train)}) and new_test ({len(new_test)})"
+
+    # Make zipped versions
+    shutil.make_archive(public / "train2", "zip", public, "train2")
+    shutil.make_archive(public / "test2", "zip", public, "test2")
+    # Remove unzipped directories (original comp doesn't have these)
+    shutil.rmtree(public / "train2")
+    shutil.rmtree(public / "test2")
+    # we also don't need the raw dirs anymore
+    shutil.rmtree(raw / "train2")
+
+    # Create answers
+    answers_df = pd.DataFrame(answer_rows)
+    answers_df.to_csv(private / "test.csv", index=False)
+
+    # Create sample submission
+    sample_submission = answers_df.copy()
+    sample_submission["probability"] = 0
+    sample_submission.to_csv(public / "sampleSubmission.csv", index=False)
+
+    assert set(answers_df.columns) == set(
+        ["clip", "probability"]
+    ), "Answers must have 'clip' and 'probability' columns"
+    assert set(sample_submission.columns) == set(
+        ["clip", "probability"]
+    ), "Sample submission must have 'clip' and 'probability' columns"

dslighting/registry/the-icml-2013-whale-challenge-right-whale-redux/prepare_val.py

@@ -0,0 +1,141 @@
+import re
+import shutil
+from pathlib import Path
+from typing import List
+
+import pandas as pd
+from tqdm import tqdm
+
+
+def _create_split(
+    train_files: List[Path],
+    test_files: List[Path],
+    public_path: Path,
+    private_path: Path,
+):
+    """
+    Helper function to process and save a single train/test split.
+
+    This function handles file copying, renaming, zipping, and the creation of
+    answer and sample submission files for a given set of train/test files.
+    """
+    # Create output directories if they don't exist
+    public_path.mkdir(exist_ok=True, parents=True)
+    private_path.mkdir(exist_ok=True, parents=True)
+
+    # Process and copy train files
+    train_output_dir = public_path / "train2"
+    train_output_dir.mkdir(exist_ok=True)
+    # Sort files to ensure deterministic indexing
+    sorted_train_files = sorted(train_files)
+    for idx, sample in enumerate(
+        tqdm(sorted_train_files, desc=f"Creating train set for {public_path.name}")
+    ):
+        new_sample_name = re.sub(r"TRAIN\d+", f"TRAIN{idx}", sample.name)
+        new_sample = train_output_dir / new_sample_name
+        shutil.copy(sample, new_sample)
+
+    # Process and copy test files, collecting answers
+    answer_rows = []
+    test_output_dir = public_path / "test2"
+    test_output_dir.mkdir(exist_ok=True)
+    # Sort files to ensure deterministic indexing
+    sorted_test_files = sorted(test_files)
+    for idx, sample in enumerate(
+        tqdm(sorted_test_files, desc=f"Creating test set for {public_path.name}")
+    ):
+        new_sample_name = sample.name.split("TRAIN")[0] + f"Test{idx}.aif"
+        new_sample = test_output_dir / new_sample_name
+        shutil.copy(sample, new_sample)
+        answer_rows.append(
+            {"clip": new_sample_name, "probability": 1 if sample.stem.endswith("_1") else 0}
+        )
+
+    # Assertions to verify file counts
+    assert len(sorted_train_files) == len(list(train_output_dir.glob("*.aif")))
+    assert len(sorted_test_files) == len(list(test_output_dir.glob("*.aif")))
+
+    # Create zipped versions and remove temporary unzipped directories
+    shutil.make_archive(public_path / "train2", "zip", public_path, "train2")
+    shutil.make_archive(public_path / "test2", "zip", public_path, "test2")
+    shutil.rmtree(train_output_dir)
+    shutil.rmtree(test_output_dir)
+
+    # Create answer file
+    answers_df = pd.DataFrame(answer_rows)
+    answers_df.to_csv(private_path / "test.csv", index=False)
+    assert set(answers_df.columns) == set(["clip", "probability"])
+
+    # Create sample submission file
+    sample_submission = answers_df.copy()
+    sample_submission["probability"] = 0
+    sample_submission.to_csv(public_path / "sampleSubmission.csv", index=False)
+    assert set(sample_submission.columns) == set(["clip", "probability"])
+
+
+def prepare(raw: Path, public: Path, private: Path):
+    """
+    Splits the data in raw into public and private datasets with appropriate test/train splits.
+    Also creates a secondary validation split in public_val/private_val directories.
+    """
+    # Data is in train2.zip - we need to unzip it
+    shutil.unpack_archive(raw / "train2.zip", raw)
+
+    # Files are named as
+    # Train: "YYYYMMDD_HHMMSS_{seconds}_TRAIN{idx}_{label:0,1}.aif"
+    # Test: "YYYYMMDD_HHMMSS_{seconds}_Test{idx}.aif"
+
+    # There are 4 days in Train and 3 days in Test
+    # In our new dataset, we'll just split Train_old into 2 days for Train and 2 days for Test
+
+    samples_by_date = {}
+    n_train_old = 0
+    for sample in (raw / "train2").iterdir():
+        date = sample.name.split("_")[0]
+        if date not in samples_by_date:
+            samples_by_date[date] = []
+        samples_by_date[date].append(sample)
+        n_train_old += 1
+
+    assert len(samples_by_date) == 4, "Expected 4 days in Train_old"
+    dates = sorted(list(samples_by_date.keys()))
+
+    # --- 1. Create the Original Split (public/private) ---
+    # This split uses the first two days for training and the last two days for testing.
+    # The outputs of this step must remain identical to the original script.
+    original_train_files = samples_by_date[dates[0]] + samples_by_date[dates[1]]
+    original_test_files = samples_by_date[dates[2]] + samples_by_date[dates[3]]
+
+    _create_split(
+        train_files=original_train_files,
+        test_files=original_test_files,
+        public_path=public,
+        private_path=private,
+    )
+
+    # --- 2. Create the New Validation Split (public_val/private_val) ---
+    # This second split takes the original *training* data (the first two days) and
+    # splits it again, using the same date-based logic. The first day becomes the
+    # new training set, and the second day becomes the new validation (test) set.
+    public_val = public.parent / "public_val"
+    private_val = private.parent / "private_val"
+
+    validation_train_files = samples_by_date[dates[0]]
+    validation_test_files = samples_by_date[dates[1]]
+
+    _create_split(
+        train_files=validation_train_files,
+        test_files=validation_test_files,
+        public_path=public_val,
+        private_path=private_val,
+    )
+
+    # Final cleanup of the raw unzipped directory after all processing is complete
+    shutil.rmtree(raw / "train2")
+
+    # Final top-level assertions from the original script
+    assert (
+        len(original_train_files) + len(original_test_files) == n_train_old
+    ), f"Expected {n_train_old} total samples in new_train ({len(original_train_files)}) and new_test ({len(original_test_files)})"
+    assert (public / "sampleSubmission.csv").exists()
+    assert (private / "test.csv").exists()

dslighting/registry/tmdb-box-office-prediction/config.yaml

@@ -0,0 +1,15 @@
+id: tmdb-box-office-prediction
+name: TMDB Box Office Prediction
+competition_type: simple
+awards_medals: false
+description: mlebench/competitions/tmdb-box-office-prediction/description.md
+
+dataset:
+  answers: tmdb-box-office-prediction/prepared/private/test.csv
+  sample_submission: tmdb-box-office-prediction/prepared/public/sample_submission.csv
+
+grader:
+  name: rmsle
+  grade_fn: mlebench.competitions.tmdb_box_office_prediction.grade:grade
+
+preparer: mlebench.competitions.tmdb_box_office_prediction.prepare:prepare

dslighting/registry/tmdb-box-office-prediction/description.md

@@ -0,0 +1,47 @@
+# TMDB Box Office Prediction
+
+## Description
+
+In this competition, you're presented with metadata on over 7,000 past films from The Movie Database to try and predict their overall worldwide box office revenue. Data points provided include cast, crew, plot keywords, budget, posters, release dates, languages, production companies, and countries.
+
+In a world where movies made an estimated $41.7 billion in 2018, the film industry is more popular than ever. But what movies make the most money at the box office? How much does a director matter? Or the budget?
+
+## Evaluation
+
+It is your job to predict the international box office revenue for each movie. For each id in the test set, you must predict the value of the revenue variable.
+
+Submissions are evaluated on Root-Mean-Squared-Logarithmic-Error (RMSLE) between the predicted value and the actual revenue:
+
+$$\text{RMSLE} = \sqrt{\frac{1}{n} \sum_{i=1}^n (\log(p_i + 1) - \log(a_i + 1))^2}$$
+
+where:
+- $n$ is the number of movies in the test set
+- $p_i$ is your predicted revenue
+- $a_i$ is the actual revenue
+- $\log(x)$ is the natural logarithm
+
+Logs are taken to not overweight blockbuster revenue movies.
+
+## Submission Format
+
+The file should contain a header and have the following format:
+
+```
+id,revenue
+1461,1000000
+1462,50000
+1463,800000000
+...
+```
+
+## Dataset Description
+
+In this dataset, you are provided with 7,398 movies and a variety of metadata obtained from The Movie Database (TMDB). Movies are labeled with id. Data points include cast, crew, plot keywords, budget, posters, release dates, languages, production companies, and countries.
+
+**Note**: Many movies are remade over the years, therefore it may seem like multiple instances of a movie may appear in the data. However, they are different and should be considered separate movies. In addition, some movies may share a title but be entirely unrelated.
+
+## Files
+
+- `train.csv`: Training data with movie metadata and revenue
+- `test.csv`: Test data with movie metadata only
+- `sample_submission.csv`: Sample submission in the correct format

dslighting/registry/tmdb-box-office-prediction/grade.py

@@ -0,0 +1,55 @@
+import numpy as np
+import pandas as pd
+from sklearn.metrics import mean_squared_log_error
+
+from mlebench.grade_helpers import InvalidSubmissionError
+
+
+def prepare_for_metric(submission: pd.DataFrame, answers: pd.DataFrame):
+    """Prepare submission and answers for RMSLE calculation."""
+    if len(submission) != len(answers):
+        raise InvalidSubmissionError(
+            f"Submission length ({len(submission)}) != answers length ({len(answers)})"
+        )
+
+    if "id" not in submission.columns:
+        raise InvalidSubmissionError("Submission must have an 'id' column")
+
+    if "revenue" not in submission.columns:
+        raise InvalidSubmissionError("Submission must have a 'revenue' column")
+
+    # Sort by id
+    submission = submission.sort_values("id").reset_index(drop=True)
+    answers = answers.sort_values("id").reset_index(drop=True)
+
+    # Check id alignment
+    if not (submission["id"] == answers["id"]).all():
+        raise InvalidSubmissionError("Submission and answers id mismatch")
+
+    # Extract predictions and true values
+    y_pred = submission["revenue"].values
+    y_true = answers["revenue"].values
+
+    # Validate predictions
+    try:
+        y_pred = y_pred.astype(float)
+    except (ValueError, TypeError):
+        raise InvalidSubmissionError("Predictions must be numeric")
+
+    if np.any(np.isnan(y_pred)):
+        raise InvalidSubmissionError("Predictions cannot contain NaN values")
+
+    if np.any(y_pred < 0):
+        raise InvalidSubmissionError("Predictions cannot be negative")
+
+    return y_true, y_pred
+
+
+def grade(submission: pd.DataFrame, answers: pd.DataFrame) -> float:
+    """Calculate RMSLE score."""
+    y_true, y_pred = prepare_for_metric(submission, answers)
+
+    # RMSLE = sqrt(MSLE)
+    rmsle = np.sqrt(mean_squared_log_error(y_true, y_pred))
+
+    return rmsle