dslighting 1.3.9__py3-none-any.whl → 1.7.6__py3-none-any.whl

dslighting/registry/dogs-vs-cats-redux-kernels-edition/prepare.py

@@ -0,0 +1,70 @@
+import shutil
+from pathlib import Path
+
+import pandas as pd
+from sklearn.model_selection import train_test_split
+from tqdm import tqdm
+
+from mlebench.utils import compress, extract, read_csv
+
+
+def prepare(raw: Path, public: Path, private: Path):
+    extract(raw / "train.zip", raw)
+    extract(raw / "test.zip", raw)
+
+    all_train_images = sorted(list((raw / "train").glob("*.jpg")))
+    # Original test ratio has Train set - 25,000 samples; Test set - 12,500 samples (33% ratio)
+    # We use 0.1 ratio to avoid removing too many samples from train
+    train_images, test_images = train_test_split(all_train_images, test_size=0.1, random_state=0)
+
+    # Copy over train images. Rename cat files to cat.0.jpg, cat.1.jpg, etc.
+    # Rename dog files to dog.0.jpg, dog.1.jpg, etc.
+    cat_ctr = 0
+    dog_ctr = 0
+    (public / "train").mkdir(exist_ok=True)
+    for img in tqdm(train_images):
+        if "cat" in img.name:
+            shutil.copy(img, public / "train" / f"cat.{cat_ctr}.jpg")
+            cat_ctr += 1
+        else:
+            shutil.copy(img, public / "train" / f"dog.{dog_ctr}.jpg")
+            dog_ctr += 1
+    assert cat_ctr + dog_ctr == len(
+        train_images
+    ), f"Expected {len(train_images)} train images but got {cat_ctr + dog_ctr} images."
+
+    # Copy over test images. Rename files to 1.jpg, 2.jpg, etc.
+    (public / "test").mkdir(exist_ok=True)
+    for i, img in enumerate(tqdm(test_images), start=1):
+        shutil.copy(img, public / "test" / f"{i}.jpg")
+    assert i == len(test_images), f"Expected {len(test_images)} test images but got {i} images."
+
+    # Compress train and test images, then remove the raw images
+    compress(public / "train", public / "train.zip", exist_ok=True)
+    compress(public / "test", public / "test.zip", exist_ok=True)
+    shutil.rmtree(raw / "train")
+    shutil.rmtree(raw / "test")
+
+    # Make answers
+    answers = pd.DataFrame(
+        {
+            "id": [i for i in range(1, len(test_images) + 1)],
+            "label": [int("dog" in img.name) for img in test_images],
+        }
+    )
+    answers.to_csv(private / "answers.csv", index=False)
+    assert len(answers) == len(
+        test_images
+    ), f"Expected {len(test_images)} answers but got {len(answers)} answers."
+
+    # Make sample submission
+    sample_submission = pd.DataFrame(
+        {
+            "id": [i for i in range(1, len(test_images) + 1)],
+            "label": [0.5 for _ in range(1, len(test_images) + 1)],
+        }
+    )
+    sample_submission.to_csv(public / "sample_submission.csv", index=False)
+    assert len(sample_submission) == len(
+        test_images
+    ), f"Expected {len(test_images)} sample submission rows but got {len(sample_submission)} rows."

dslighting/registry/dogs-vs-cats-redux-kernels-edition/prepare_val.py

@@ -0,0 +1,143 @@
+import shutil
+from pathlib import Path
+from typing import List
+
+import pandas as pd
+from sklearn.model_selection import train_test_split
+from tqdm import tqdm
+
+from mlebench.utils import compress, extract
+
+
+def _process_split(
+    image_paths: List[Path],
+    public_dir: Path,
+    private_dir: Path,
+    test_size: float,
+    random_state: int,
+) -> List[Path]:
+    """
+    Splits a list of image paths into train/test sets and generates all required files.
+
+    This function encapsulates the logic for:
+    1. Splitting data.
+    2. Copying and renaming images to public train/test subdirectories.
+    3. Compressing the public train/test image directories.
+    4. Creating the private answer key for the test set.
+    5. Creating a public sample submission file.
+
+    Args:
+        image_paths: A list of Path objects for the images to be split.
+        public_dir: The public output directory (e.g., 'data/public').
+        private_dir: The private output directory (e.g., 'data/private').
+        test_size: The proportion of the dataset to allocate to the test set.
+        random_state: The seed for the random number generator.
+
+    Returns:
+        A list of Path objects corresponding to the training set of this split.
+    """
+    public_dir.mkdir(exist_ok=True, parents=True)
+    private_dir.mkdir(exist_ok=True, parents=True)
+
+    # Perform the split
+    train_images, test_images = train_test_split(
+        image_paths, test_size=test_size, random_state=random_state
+    )
+
+    # Copy over train images. Rename cat files to cat.0.jpg, cat.1.jpg, etc.
+    # Rename dog files to dog.0.jpg, dog.1.jpg, etc.
+    cat_ctr = 0
+    dog_ctr = 0
+    (public_dir / "train").mkdir(exist_ok=True)
+    for img in tqdm(train_images, desc=f"Processing train set for {public_dir.name}"):
+        if "cat" in img.name:
+            shutil.copy(img, public_dir / "train" / f"cat.{cat_ctr}.jpg")
+            cat_ctr += 1
+        else:
+            shutil.copy(img, public_dir / "train" / f"dog.{dog_ctr}.jpg")
+            dog_ctr += 1
+    assert cat_ctr + dog_ctr == len(
+        train_images
+    ), f"Expected {len(train_images)} train images but got {cat_ctr + dog_ctr} images."
+
+    # Copy over test images. Rename files to 1.jpg, 2.jpg, etc.
+    (public_dir / "test").mkdir(exist_ok=True)
+    for i, img in enumerate(tqdm(test_images, desc=f"Processing test set for {public_dir.name}"), start=1):
+        shutil.copy(img, public_dir / "test" / f"{i}.jpg")
+    assert i == len(test_images), f"Expected {len(test_images)} test images but got {i} images."
+
+    # Compress train and test images
+    compress(public_dir / "train", public_dir / "train.zip", exist_ok=True)
+    compress(public_dir / "test", public_dir / "test.zip", exist_ok=True)
+
+    # Make answers
+    answers = pd.DataFrame(
+        {
+            "id": [i for i in range(1, len(test_images) + 1)],
+            "label": [int("dog" in img.name) for img in test_images],
+        }
+    )
+    answers.to_csv(private_dir / "answers.csv", index=False)
+    assert len(answers) == len(
+        test_images
+    ), f"Expected {len(test_images)} answers but got {len(answers)} answers."
+
+    # Make sample submission
+    sample_submission = pd.DataFrame(
+        {
+            "id": [i for i in range(1, len(test_images) + 1)],
+            "label": [0.5 for _ in range(1, len(test_images) + 1)],
+        }
+    )
+    sample_submission.to_csv(public_dir / "sample_submission.csv", index=False)
+    assert len(sample_submission) == len(
+        test_images
+    ), f"Expected {len(test_images)} sample submission rows but got {len(sample_submission)} rows."
+
+    return train_images
+
+
+def prepare(raw: Path, public: Path, private: Path):
+    # This part remains from the original script
+    extract(raw / "train.zip", raw)
+    extract(raw / "test.zip", raw)
+
+    all_train_images = sorted(list((raw / "train").glob("*.jpg")))
+    
+    # --- 1. Original Split (train -> train + test) ---
+    # This call generates the original competition files in `public` and `private`.
+    # The logic and outputs of this step are unchanged.
+    # Original test ratio has Train set - 25,000 samples; Test set - 12,500 samples (33% ratio)
+    # We use 0.1 ratio to avoid removing too many samples from train
+    original_test_size = 0.1
+    main_train_set = _process_split(
+        image_paths=all_train_images,
+        public_dir=public,
+        private_dir=private,
+        test_size=original_test_size,
+        random_state=0,
+    )
+
+    # --- 2. New Validation Split (main_train_set -> train_val + test_val) ---
+    # This call generates a new, parallel set of files for validation purposes.
+    # It operates *only* on the training data from the first split.
+    public_val = public.parent / "public_val"
+    private_val = private.parent / "private_val"
+
+    # To make the new test_val set have the same size as the original test set,
+    # we adjust the test_size for the second split.
+    # new_size = original_test_size / (1 - original_test_size)
+    # e.g., 0.1 / (1 - 0.1) = 0.1 / 0.9 = 1/9
+    validation_test_size = original_test_size / (1 - original_test_size)
+
+    _process_split(
+        image_paths=main_train_set,
+        public_dir=public_val,
+        private_dir=private_val,
+        test_size=validation_test_size,
+        random_state=0,  # Use the same random state for consistency
+    )
+
+    # Final cleanup is done after all splits are complete
+    shutil.rmtree(raw / "train")
+    shutil.rmtree(raw / "test")

dslighting/registry/ethanol-concentration/checksums.yaml

@@ -0,0 +1,3 @@
+# Checksums for ethanol-concentration dataset
+# This file will be auto-generated during data preparation
+zip: ""

dslighting/registry/ethanol-concentration/config.yaml

@@ -0,0 +1,16 @@
+id: ethanol-concentration
+name: Ethanol Concentration Time Series Classification
+competition_type: code
+awards_medals: false
+prizes: null
+description: mlebench/competitions/ethanol-concentration/description.md
+
+dataset:
+  answers: ethanol-concentration/prepared/private/test_labels.csv
+  sample_submission: ethanol-concentration/prepared/public/sample_submission.csv
+
+grader:
+  name: accuracy
+  grade_fn: mlebench.competitions.ethanol-concentration.grade:grade
+
+preparer: mlebench.competitions.ethanol-concentration.prepare:prepare

dslighting/registry/ethanol-concentration/description.md

@@ -0,0 +1,64 @@
+# Ethanol Concentration Time Series Classification
+
+## Task Description
+
+You are solving a machine learning task of time series classification using the Ethanol Concentration dataset.
+
+The dataset presented here comprises real-world time series data. We have split the dataset into train and test parts.
+
+## Data Description
+
+- **Input**: A sequence of observed features (SEQUENCE_LENGTH=1751, FEATURE_DIM=3)
+- **Output**: Predict the labels for the given sequence, where the label is in range of {0, 1, 2, 3} (4 classes total)
+- **Evaluation Metric**: Accuracy
+
+## Files
+
+- `train_data.npy` - Training features in numpy format (shape: [N_train, 1751, 3])
+- `train_labels.npy` - Training labels in numpy format (shape: [N_train,])
+- `test_data.npy` - Test features in numpy format (shape: [N_test, 1751, 3])
+- `sample_submission.csv` - Sample submission file showing the required format
+
+## Data Loading
+
+You can load the data using numpy:
+
+```python
+import numpy as np
+
+# Load training data
+X_train = np.load('train_data.npy')
+y_train = np.load('train_labels.npy')
+
+# Load test data
+X_test = np.load('test_data.npy')
+
+print(f"X_train shape: {X_train.shape}")  # (N_train, 1751, 3)
+print(f"y_train shape: {y_train.shape}")  # (N_train,)
+print(f"X_test shape: {X_test.shape}")    # (N_test, 1751, 3)
+```
+
+## Submission Format
+
+The submission file should contain predictions for each test sample:
+- Column `id`: Integer index for each test sample
+- Column `label`: Predicted class label (integer in range [0, 3])
+
+Example:
+```
+id,label
+0,2
+1,0
+2,3
+...
+```
+
+## Goal
+
+Train a time series classification model to achieve good performance on the test set as measured by accuracy.
+
+## Notes
+
+- The dataset contains real-world time series data for ethanol concentration classification
+- Each sample is a multivariate time series with 3 features observed over 1751 time steps
+- There are 4 distinct classes to predict

dslighting/registry/ethanol-concentration/grade.py

@@ -0,0 +1,23 @@
+import pandas as pd
+import numpy as np
+from sklearn.metrics import accuracy_score
+
+
+def grade(submission: pd.DataFrame, answers: pd.DataFrame) -> float:
+    """
+    Grade the submission using accuracy metric.
+
+    Args:
+        submission: DataFrame with columns ['id', 'label']
+        answers: DataFrame with columns ['id', 'label']
+
+    Returns:
+        Accuracy score (float between 0 and 1)
+    """
+    # Merge on id to ensure alignment
+    merged = pd.merge(answers, submission, on='id', suffixes=('_true', '_pred'))
+
+    # Calculate accuracy
+    accuracy = accuracy_score(merged['label_true'], merged['label_pred'])
+
+    return accuracy

dslighting/registry/ethanol-concentration/prepare.py

@@ -0,0 +1,90 @@
+import pandas as pd
+import numpy as np
+import sys
+from pathlib import Path
+
+
+def prepare(raw: Path, public: Path, private: Path):
+    """
+    Prepare the ethanol-concentration dataset for the benchmark.
+
+    This function converts the .ts time series files to numpy arrays,
+    so that public directory only contains data, not data loading code.
+
+    Args:
+        raw: Path to raw data directory (contains EthanolConcentration_TRAIN.ts and EthanolConcentration_TEST.ts)
+        public: Path to public directory (visible to participants)
+        private: Path to private directory (hidden from participants, used for grading)
+    """
+    # Load test data to extract labels for grading
+    try:
+        # Add raw directory to path temporarily to import dataset module
+        sys.path.insert(0, str(raw))
+
+        # Use the local dataset module to load data
+        from dataset import get_dataset
+
+        # Change to raw directory to load data
+        import os
+        original_dir = os.getcwd()
+        os.chdir(str(raw))
+
+        try:
+            # Load train and test datasets
+            X_train, y_train = get_dataset('TRAIN')
+            X_test, y_test = get_dataset('TEST')
+
+            print(f"Loaded training data: X_train.shape = {X_train.shape}, y_train.shape = {y_train.shape}")
+            print(f"Loaded test data: X_test.shape = {X_test.shape}, y_test.shape = {y_test.shape}")
+            print(f"Number of unique labels: {len(np.unique(y_train))}")
+
+            # Save training data as numpy arrays in public directory
+            np.save(public / "train_data.npy", X_train)
+            np.save(public / "train_labels.npy", y_train.flatten())
+
+            # Save test data (without labels) in public directory
+            np.save(public / "test_data.npy", X_test)
+
+            # Create test labels dataframe for grading (private)
+            test_labels_df = pd.DataFrame(
+                {"id": range(len(y_test)), "label": y_test.flatten()}
+            )
+            test_labels_df.to_csv(private / "test_labels.csv", index=False)
+
+            # Create sample submission file
+            sample_submission = pd.DataFrame(
+                {"id": range(len(y_test)), "label": 0}  # Default to class 0
+            )
+            sample_submission.to_csv(public / "sample_submission.csv", index=False)
+
+            print(f"Data preparation completed:")
+            print(f"  - Training: {len(X_train)} samples")
+            print(f"  - Test: {len(X_test)} samples")
+            print(f"  - Sequence length: {X_train.shape[1]}")
+            print(f"  - Feature dimension: {X_train.shape[2]}")
+            print(f"  - Number of classes: {len(np.unique(y_train))}")
+
+        finally:
+            os.chdir(original_dir)
+            sys.path.remove(str(raw))
+
+    except Exception as e:
+        print(f"Error loading test labels: {e}")
+        import traceback
+        traceback.print_exc()
+        # Fallback: create dummy files if loading fails
+        print("Creating dummy submission files...")
+        dummy_df = pd.DataFrame({"id": [0], "label": [0]})
+        dummy_df.to_csv(private / "test_labels.csv", index=False)
+        dummy_df.to_csv(public / "sample_submission.csv", index=False)
+
+    # Validation checks
+    assert (public / "train_data.npy").exists(), "Training data should exist"
+    assert (public / "train_labels.npy").exists(), "Training labels should exist"
+    assert (public / "test_data.npy").exists(), "Test data should exist"
+    assert (private / "test_labels.csv").exists(), "Test labels should exist"
+    assert (public / "sample_submission.csv").exists(), "Sample submission should exist"
+
+    print(f"\nPrepared ethanol-concentration dataset:")
+    print(f"  - Public files: {list(public.glob('*'))}")
+    print(f"  - Private files: {list(private.glob('*'))}")

dslighting/registry/facebook-recruiting-iii-keyword-extraction/checksums.yaml

@@ -0,0 +1,7 @@
+private:
+  test.csv: 1aeaa15b5bce7088a7d5423c2c404eca
+public:
+  sample_submission.csv: e4ea6ff928dff4bf2d2a5c3b0089402c
+  test.csv: 10335b0f34418bb6a523506290aff802
+  train.csv: 3e16dc3e4df1697f70c93cf6bb473eaf
+zip: 68ceda9a963e745ab4fe02d0939638f7

dslighting/registry/facebook-recruiting-iii-keyword-extraction/config.yaml

@@ -0,0 +1,16 @@
+id: facebook-recruiting-iii-keyword-extraction
+name: Facebook Recruiting III - Keyword Extraction
+competition_type: simple
+awards_medals: true
+prizes: null
+description: mlebench/competitions/facebook-recruiting-iii-keyword-extraction/description.md
+
+dataset:
+  answers: facebook-recruiting-iii-keyword-extraction/prepared/private/test.csv
+  sample_submission: facebook-recruiting-iii-keyword-extraction/prepared/public/sample_submission.csv
+
+grader:
+  name: micro-f1-score
+  grade_fn: mlebench.competitions.facebook-recruiting-iii-keyword-extraction.grade:grade
+
+preparer: mlebench.competitions.facebook-recruiting-iii-keyword-extraction.prepare:prepare

dslighting/registry/facebook-recruiting-iii-keyword-extraction/description.md

@@ -0,0 +1,74 @@
+# Overview
+
+## Description
+
+Looking for a data science position at Facebook?  After two successful prior Kaggle competitions, Facebook continues their mission to identify the best data scientists and software engineers that Kaggle has to offer. In this third installment, they seek candidates who have experience text mining large amounts of data.
+
+![tags.jpg](https://storage.googleapis.com/kaggle-media/competitions/kaggle/3539/media/tags.jpg)
+
+This competition tests your text skills on a large dataset from the Stack Exchange sites.  The task is to predict the tags (a.k.a. keywords, topics, summaries), given only the question text and its title. The dataset contains content from disparate stack exchange sites, containing a mix of both technical and non-technical questions.
+
+*Positions are available in Menlo Park, Seattle, New York City, and London; candidates must have, or be eligible to obtain, authorization to work in the US or UK.*
+
+**Please note: you must compete as an individual in recruiting competitions. You may only use the data provided to make your predictions. Crawling stack exchange sites to look up answers is not permitted. Facebook will review the code of the top participants before deciding whether to offer an interview.**
+
+**This competition counts towards rankings & achievements.**  If you wish to be considered for an interview at Facebook, check the box "Allow host to contact me" when you make your first entry.
+
+### Acknowledgments
+
+We thank Stack Exchange (and its users) for generously releasing the source dataset through its [Creative Commons Data Dumps](http://blog.stackoverflow.com/category/cc-wiki-dump). All data is licensed under the [cc-by-sa](http://creativecommons.org/licenses/by-sa/2.5/) license.
+
+## Prizes
+
+Highly ranked contestants who indicate their interest will be considered by Facebook for interviews, based on their work in the competition and additional background. Positions are available in Menlo Park, Seattle, New York City, and London. Candidates must have, or be eligible to obtain, authorization to work in the US or UK.
+
+## Evaluation
+
+The evaluation metric for this competition is [Mean F1-Score](https://www.kaggle.com/wiki/MeanFScore). The F1 score, commonly used in information retrieval, measures accuracy using the statistics precision p and recall r. Precision is the ratio of true positives (tp) to all predicted positives (tp + fp). Recall is the ratio of true positives to all actual positives (tp + fn). The F1 score is given by:
+
+$F_1 = 2 \frac{p * r}{p + r}$ where $p = \frac{tp}{tp + fp}$ and $r = \frac{tp}{tp + fn}$
+
+The F1 metric weights recall and precision equally, and a good retrieval algorithm will maximize both precision and recall simultaneously. Thus, moderately good performance on both will be favored over extremely good performance on one and poor performance on the other.
+
+To receive credit, the tag you predict must be an exact match, regardless of whether you believe the tags are synonyms. Why aren't synonyms counted?
+
+1. Giving out a list of candidate synonyms is a potential source of [leakage](https://www.kaggle.com/wiki/Leakage)
+2. Synonyms are subjective, and there are "subjectively many" synonyms for a given tag
+3. Participants are equally penalized for predicting a synonym of a correct tag, so the task can be framed as not only predicting a tag, but also modeling the distribution(s) of potential synonyms
+
+### Submission Instructions
+
+For every question in the test set, your submission file should contain two columns: Id and Tags. Id is the unique identifier of a question in test.csv. Tags should be a space-delimited list predicted tags. You should maintain the order of questions.
+
+The file should contain a header and have the following format:
+
+```
+Id,Tags
+1,"c++ javaScript"
+2,"php python mysql"
+3,"django"
+etc.
+```
+
+## Citation
+
+Alex Hollander, Anirudh Ranganath, Itamar Rosenn, Jim Dullaghan, John P. Costella, Junfeng Pan, Will Cukierski. (2013). Facebook Recruiting III - Keyword Extraction. Kaggle. https://kaggle.com/competitions/facebook-recruiting-iii-keyword-extraction
+
+# Dataset Description
+
+All of the data is in 2 files: Train and Test.
+
+**Train.csv** contains 4 columns: Id,Title,Body,Tags
+
+- Id - Unique identifier for each question
+- Title - The question's title
+- Body - The body of the question
+- Tags - The tags associated with the question (all lowercase, should not contain tabs '\t' or ampersands '&')
+
+**Test.csv** contains the same columns but without the Tags, which you are to predict.
+
+The questions are randomized and contains a mix of verbose text sites as well as sites related to math and programming. The number of questions from each site may vary, and no filtering has been performed on the questions (such as closed questions).
+
+## Acknowledgements
+
+We thank Stack Exchange (and its users) for generously releasing the source dataset through its [Creative Commons Data Dumps](http://blog.stackoverflow.com/category/cc-wiki-dump). All data is licensed under the [cc-by-sa](http://creativecommons.org/licenses/by-sa/2.5/) license.

dslighting/registry/facebook-recruiting-iii-keyword-extraction/description_obfuscated.md

@@ -0,0 +1,33 @@
+
+# Task
+
+The task is to predict tags (a.k.a. keywords, topics, summaries), given only Stack Exchange question text and its title.
+
+# Metric
+
+Mean F1-Score.
+
+# Submission Format
+
+For every question in the test set, your submission file should contain two columns: Id and Tags. Id is the unique identifier of a question in test.csv. Tags should be a space-delimited list predicted tags. You should maintain the order of questions.
+
+The file should contain a header and have the following format:
+
+```
+Id,Tags
+1,"c++ javaScript"
+2,"php python mysql"
+3,"django"
+etc.
+```
+
+# Dataset 
+
+**Train.csv** contains 4 columns: Id,Title,Body,Tags
+
+- Id - Unique identifier for each question
+- Title - The question's title
+- Body - The body of the question
+- Tags - The tags associated with the question (all lowercase, should not contain tabs '\t' or ampersands '&')
+
+**Test.csv** contains the same columns but without the Tags, which you are to predict.

dslighting/registry/facebook-recruiting-iii-keyword-extraction/grade.py

@@ -0,0 +1,60 @@
+import pandas as pd
+from pandas import DataFrame
+from scipy.sparse import csr_matrix
+from sklearn.metrics import f1_score
+from sklearn.preprocessing import MultiLabelBinarizer
+
+from mlebench.grade_helpers import InvalidSubmissionError
+from mlebench.utils import get_logger
+
+logger = get_logger(__name__)
+
+
+def grade(submission: DataFrame, answers: DataFrame) -> float:
+    """Grades the submission against the test set."""
+
+    y_true, y_pred = prepare_for_metric(submission, answers)
+    return f1_score(y_true=y_true, y_pred=y_pred, average="micro")
+
+
+def prepare_for_metric(
+    submission: pd.DataFrame, answers: pd.DataFrame
+) -> tuple[csr_matrix, csr_matrix]:
+
+    # Answer checks
+    assert "Id" in answers.columns, "Answers must have an 'Id' column."
+    assert "Tags" in answers.columns, "Answers must have a 'Tags' column."
+
+    # Submission checks
+    if "Id" not in submission.columns:
+        raise InvalidSubmissionError("Submission DataFrame must have an 'Id' column.")
+    if "Tags" not in submission.columns:
+        raise InvalidSubmissionError("Submission DataFrame must have a 'Tags' column.")
+    if len(submission) != len(answers):
+        raise InvalidSubmissionError(
+            "Submission DataFrame must have the same number of rows as the answers DataFrame."
+        )
+
+    # Match order
+    submission = submission.sort_values(by="Id").reset_index(drop=True)
+    answers = answers.sort_values(by="Id").reset_index(drop=True)
+    if (submission["Id"].values != answers["Id"].values).any():
+        raise InvalidSubmissionError("Submission and answers must have matching 'Id's.")
+
+    # Get classes
+    classes = set()
+
+    for tags in answers["Tags"]:
+        if not isinstance(tags, str):
+            logger.warning(f"Tags from answers '{tags}' not of type str! Skipping.")
+            continue
+
+        tags_split = tags.split()
+        classes.update(tags_split)
+
+    # Convert to sparse matrix using MultiLabelBinarizer
+    mlb = MultiLabelBinarizer(classes=sorted(classes), sparse_output=True)
+    y_true = mlb.fit_transform(answers["Tags"].fillna("").str.split())
+    y_pred = mlb.transform(submission["Tags"].fillna("").str.split())
+
+    return y_true, y_pred

dslighting/registry/facebook-recruiting-iii-keyword-extraction/prepare.py

@@ -0,0 +1,41 @@
+from pathlib import Path
+
+from sklearn.model_selection import train_test_split
+
+from mlebench.utils import extract, get_logger, read_csv
+
+logger = get_logger(__name__)
+
+
+def prepare(raw: Path, public: Path, private: Path) -> None:
+    extract(raw / "Train.zip", raw)
+
+    # Create new train and test from original train set
+    logger.info(f"Creating new train and test sets from {raw}.")
+
+    old_train = read_csv(raw / "Train.csv", dtype={"Id": str, "Tags": str})
+    new_train, new_test = train_test_split(old_train, test_size=0.1, random_state=0)
+    new_test_without_labels = new_test.drop(columns=["Tags"])
+    sample_submission = new_test_without_labels.copy()
+    sample_submission["Tags"] = "javascript c# python php java"
+
+    # Copy over files to private and public directories
+    logger.info(f"Copying files to {private} and {public}.")
+
+    new_train.to_csv(public / "train.csv", index=False)
+    new_test_without_labels.to_csv(public / "test.csv", index=False)
+    sample_submission.to_csv(public / "sample_submission.csv", index=False)
+    new_test.to_csv(private / "test.csv", index=False)
+
+    # Sanity checks
+    logger.info("Performing sanity checks.")
+
+    assert len(new_test_without_labels) == len(
+        new_test
+    ), f"Expected {len(new_test)} public test samples, got {len(new_test_without_labels)}."
+    assert len(old_train) == len(new_train) + len(
+        new_test
+    ), f"Mismatch in number of samples in new train and test split! Expected {len(old_train)} samples, got {len(new_train) + len(new_test)}."
+    assert len(sample_submission) == len(
+        new_test
+    ), f"Expected {len(new_test)} public test samples, got {len(sample_submission)}."