dslighting 1.3.9__py3-none-any.whl → 1.7.1__py3-none-any.whl

dslighting/registry/tabular-playground-series-may-2022/prepare.py

@@ -0,0 +1,56 @@
+from pathlib import Path
+
+import numpy as np
+from sklearn.model_selection import train_test_split
+
+from mlebench.utils import read_csv
+
+
+def prepare(raw: Path, public: Path, private: Path):
+
+    old_train = read_csv(raw / "train.csv")
+
+    # 900k train, 1.6m - 900k = 700k test; so 700k/1.6m = 0.4375
+    # We create our split at 100,000 test samples to get same OOM while keeping as many samples as possible in train
+    new_train, new_test = train_test_split(old_train, test_size=100_000, random_state=0)
+
+    # make ids go from 0 to len(new_train) - 1
+    new_train.id = np.arange(len(new_train))
+    # and from len(new_train) to len(new_train) + len(new_test) - 1
+    new_test.id = np.arange(len(new_train), len(new_train) + len(new_test))
+
+    # make downstream files
+    new_test_without_labels = new_test.drop(columns=["target"]).copy()
+    gold_submission = new_test[["id", "target"]].copy()
+    sample_submission = gold_submission.copy()
+    sample_submission.target = 0.5
+
+    # save
+    new_train.to_csv(public / "train.csv", index=False)
+    new_test.to_csv(private / "test.csv", index=False)
+    new_test_without_labels.to_csv(public / "test.csv", index=False)
+    gold_submission.to_csv(private / "gold_submission.csv", index=False)
+    sample_submission.to_csv(public / "sample_submission.csv", index=False)
+
+    # checks
+    assert len(new_train) + len(new_test) == len(
+        old_train
+    ), "Expected the sum of the lengths of the new train and test to be equal to the length of the original train."
+    assert len(new_test) == len(
+        sample_submission
+    ), "Expected the length of the private test to be equal to the length of the sample submission."
+    assert len(new_test) == len(
+        gold_submission
+    ), "Expected the length of the private test to be equal to the length of the gold submission."
+
+    assert (
+        new_train.columns.to_list() == old_train.columns.to_list()
+    ), "Expected the columns of the new train to be the same as the columns of the original train."
+    assert (
+        new_test.columns.to_list() == old_train.columns.to_list()
+    ), "Expected the columns of the new test to be the same as the columns of the original train"
+
+    # check that ids dont overlap between train and test
+    assert set(new_train.id).isdisjoint(
+        set(new_test.id)
+    ), "Expected the ids of the new train and test to be disjoint."

dslighting/registry/tabular-playground-series-may-2022/prepare_val.py

@@ -0,0 +1,116 @@
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+from sklearn.model_selection import train_test_split
+
+from mlebench.utils import read_csv
+
+
+def _create_and_save_split(
+    df_to_split: pd.DataFrame,
+    public_path: Path,
+    private_path: Path,
+    test_size: int,
+    random_state: int,
+):
+    """
+    Helper function to perform a data split, re-index, and save all required files.
+
+    Args:
+        df_to_split (pd.DataFrame): The dataframe to be split.
+        public_path (Path): The directory for public-facing files (train, test features).
+        private_path (Path): The directory for private-facing files (test labels).
+        test_size (int): The number of samples for the test set.
+        random_state (int): The random state for reproducibility.
+
+    Returns:
+        pd.DataFrame: The new, smaller training dataframe from the split.
+    """
+    # Create output directories if they don't exist
+    public_path.mkdir(parents=True, exist_ok=True)
+    private_path.mkdir(parents=True, exist_ok=True)
+
+    # Perform the split
+    train_df, test_df = train_test_split(
+        df_to_split, test_size=test_size, random_state=random_state
+    )
+
+    # make ids go from 0 to len(train_df) - 1
+    train_df.id = np.arange(len(train_df))
+    # and from len(train_df) to len(train_df) + len(test_df) - 1
+    test_df.id = np.arange(len(train_df), len(train_df) + len(test_df))
+
+    # make downstream files
+    test_df_without_labels = test_df.drop(columns=["target"]).copy()
+    gold_submission = test_df[["id", "target"]].copy()
+    sample_submission = gold_submission.copy()
+    sample_submission.target = 0.5
+
+    # save files with required names
+    train_df.to_csv(public_path / "train.csv", index=False)
+    test_df.to_csv(private_path / "test.csv", index=False)
+    test_df_without_labels.to_csv(public_path / "test.csv", index=False)
+    gold_submission.to_csv(private_path / "gold_submission.csv", index=False)
+    sample_submission.to_csv(public_path / "sample_submission.csv", index=False)
+
+    # run checks for this split
+    assert len(train_df) + len(test_df) == len(
+        df_to_split
+    ), "Expected the sum of the lengths of the new train and test to be equal to the length of the input data."
+    assert len(test_df) == len(
+        sample_submission
+    ), "Expected the length of the private test to be equal to the length of the sample submission."
+    assert len(test_df) == len(
+        gold_submission
+    ), "Expected the length of the private test to be equal to the length of the gold submission."
+    assert (
+        train_df.columns.to_list() == df_to_split.columns.to_list()
+    ), "Expected the columns of the new train to be the same as the columns of the input data."
+    assert (
+        test_df.columns.to_list() == df_to_split.columns.to_list()
+    ), "Expected the columns of the new test to be the same as the columns of the input data"
+    assert set(train_df.id).isdisjoint(
+        set(test_df.id)
+    ), "Expected the ids of the new train and test to be disjoint."
+
+    return train_df
+
+
+def prepare(raw: Path, public: Path, private: Path):
+
+    old_train = read_csv(raw / "train.csv")
+
+    # --- Stage 1: Create the original train/test split ---
+    # This section produces the primary competition data. Its outputs in `public/`
+    # and `private/` are guaranteed to be identical to the original script.
+
+    # 900k train, 1.6m - 900k = 700k test; so 700k/1.6m = 0.4375
+    # We create our split at 100,000 test samples to get same OOM while keeping as many samples as possible in train
+    new_train = _create_and_save_split(
+        df_to_split=old_train,
+        public_path=public,
+        private_path=private,
+        test_size=100_000,
+        random_state=0,
+    )
+
+    # --- Stage 2: Create the new train/validation split ---
+    # This section takes the `new_train` set from Stage 1 and splits it again
+    # to create a smaller training set and a new validation set.
+    # The outputs are saved to parallel `public_val/` and `private_val/` directories.
+
+    # Define paths for the new validation set directories, parallel to the original ones.
+    public_val = public.parent / "public_val"
+    private_val = private.parent / "private_val"
+
+    # Split `new_train` again using the exact same logic and test size.
+    # The resulting `test_val` set will have 100,000 samples, same as the original `test` set.
+    # The filenames and directory structure are mirrored for consistency.
+    _create_and_save_split(
+        df_to_split=new_train,
+        public_path=public_val,
+        private_path=private_val,
+        test_size=100_000,
+        random_state=0,
+    )

dslighting/registry/tensorflow-speech-recognition-challenge/checksums.yaml

@@ -0,0 +1,5 @@
+private:
+  test.csv: 5f4d739491ce2cc9acecaffca9787367
+public:
+  sample_submission.csv: 0b22e4798056a81416614ee8aa3c6ae5
+zip: 8e459afaaf22efd6411d545a690a7452

dslighting/registry/tensorflow-speech-recognition-challenge/config.yaml

@@ -0,0 +1,24 @@
+id: tensorflow-speech-recognition-challenge
+name: TensorFlow Speech Recognition Challenge
+competition_type: simple
+awards_medals: true
+prizes:
+  - position: 1
+    value: 8000
+  - position: 2
+    value: 6000
+  - position: 3
+    value: 3000
+  - position: Special TensorFlow Prize
+    value: 8000
+description: mlebench/competitions/tensorflow-speech-recognition-challenge/description.md
+
+dataset:
+  answers: tensorflow-speech-recognition-challenge/prepared/private/test.csv
+  sample_submission: tensorflow-speech-recognition-challenge/prepared/public/sample_submission.csv
+
+grader:
+  name: accuracy
+  grade_fn: mlebench.competitions.tensorflow-speech-recognition-challenge.grade:grade
+
+preparer: mlebench.competitions.tensorflow-speech-recognition-challenge.prepare:prepare

dslighting/registry/tensorflow-speech-recognition-challenge/description.md

@@ -0,0 +1,328 @@
+# Overview
+
+## Description
+
+We might be on the verge of too many screens. It seems like everyday, new versions of common objects are “re-invented” with built-in wifi and bright touchscreens. A promising antidote to our screen addiction are voice interfaces.
+
+![lilly-rum-342740.jpg](https://storage.googleapis.com/kaggle-media/competitions/tensorflow/lilly-rum-342740.jpg)
+
+But, for independent makers and entrepreneurs, it’s hard to build a simple speech detector using free, open data and code. Many voice recognition datasets require preprocessing before a neural network model can be built on them. To help with this, [TensorFlow](https://www.tensorflow.org/) recently released the Speech Commands Datasets. It includes 65,000 one-second long utterances of 30 short words, by thousands of different people.
+
+In this competition, you're challenged to use the Speech Commands Dataset to build an algorithm that understands simple spoken commands. By improving the recognition accuracy of open-sourced voice interface tools, we can improve product effectiveness and their accessibility.
+
+## Evaluation
+
+Submissions are evaluated on Multiclass Accuracy, which is simply the average number of observations with the correct label.
+
+**Note:** There are only 12 possible labels for the Test set: `yes`, `no`, `up`, `down`, `left`, `right`, `on`, `off`, `stop`, `go`, `silence`, `unknown`.
+
+The `unknown` label should be used for a command that is not one one of the first 10 labels or that is not `silence`.
+
+### Submission File
+
+For audio clip in the test set, you must predict the correct `label`. The submission file should contain a header and have the following format:
+
+```
+fname,label
+clip_000044442.wav,silence
+clip_0000adecb.wav,left
+clip_0000d4322.wav,unknown
+etc.
+```
+
+## Prizes
+
+**Leaderboard Prizes**:
+
+- 1st place - $8,000
+- 2nd place - $6,000
+- 3rd place - $3,000
+
+**Special TensorFlow Prize**: $8,000
+
+**UPDATE 01/17/2018**: Information about being considered for this prize can be found on [this forum post](https://www.kaggle.com/c/tensorflow-speech-recognition-challenge/discussion/47703).
+
+The goal of the special prize is to encourage contestants to create a model that can be useful in practice to recognize commands on a Raspberry Pi 3. In order to do this, the models must:
+
+- Be runnable as frozen TensorFlow GraphDef files with no additional dependencies beyond TensorFlow 1.4.
+- Be small in size (below 5,000,000 bytes).
+- Have a standard set of inputs and outputs:
+    - decoded_sample_data:0, taking a [16000, 1] float tensor as input, representing the audio PCM-encoded data.
+    - decoded_sample_data:1, taking a scalar [] int32 tensor as input, representing the sample rate, which must be the value 16000.
+    - labels_softmax:0, a [12] float tensor representing the probabilities of each class label as an output, from zero to one.
+- Run in less than 200ms on a stock Raspberry Pi 3 running Raspbian GNU/Linux 8 (Jessie), with no overclocking.
+- Must come with code to train the model, which must be license-compatible with TensorFlow (Apache), and be submittable through Google’s CLA to the TensorFlow project.
+
+Eligibility for the Special Prize will be qualified by the host by re-running your solution within his environment. The model evaluated must match the one generating the predictions scored on the leaderboard. At the close of the competition, you will have the ability to self-select for consideration as a special prize winner. Due to possible inherent discrepancies in environments, it's recommended that you target a latency closer to 175ms or faster. For calibration purposes, the host has provided:
+
+To evaluate the latency of each model, we’ll be running the following command:
+
+```
+./benchmark_model --graph=<Your Graph> --input_layer="decoded_sample_data:0,decoded_sample_data:1" --input_layer_shape="16000,1:" --input_layer_type="float,int32" --input_layer_values=":16000" --output_layer="labels_softmax:0" --show_run_order=false --show_time=false --show_memory=false --show_summary=true --show_flops=true
+
+```
+
+The result will be the ‘avg=’ value shown in the line with the prefix ‘Timings (microseconds)’ near the end of the logging. This represents the mean time in microseconds of repeated runs by the command. To be eligible for the prize, the result must be below 200,000 microseconds (200 milliseconds). Because of variability in timings, I recommend that you aim for 175ms to have a margin of safety.
+
+To reproduce the process on your own Raspberry Pi 3, here’s how to download the benchmark binary and run it on a pretrained graph produced by the training tutorial:
+
+```
+curl -O https://storage.googleapis.com/download.tensorflow.org/models/speech_commands_v0.01.zip
+unzip speech_commands_v0.01.zip
+
+```
+
+```
+curl -O https://storage.googleapis.com/download.tensorflow.org/deps/pi/2017_10_07/benchmark_model
+chmod +x benchmark_model
+./benchmark_model --graph=conv_actions_frozen.pb --input_layer="decoded_sample_data:0,decoded_sample_data:1" --input_layer_shape="16000,1:" --input_layer_type="float,int32" --input_layer_values=":16000" --output_layer="labels_softmax:0" --show_run_order=false --show_time=false --show_memory=false --show_summary=true --show_flops=true
+```
+
+## Timeline
+
+- **January 9, 2018** - Entry deadline. You must accept the competition rules before this date in order to compete.
+- **January 9, 2018** - Team Merger deadline. This is the last day participants may join or merge teams.
+- **January 16, 2018** - Final submission deadline.
+
+All deadlines are at 11:59 PM UTC on the corresponding day unless otherwise noted. The competition organizers reserve the right to update the contest timeline if they deem it necessary.
+
+## Tutorials & More Info
+
+The host has provided the following resources:
+
+- [Google Research Blog Post](https://research.googleblog.com/2017/08/launching-speech-commands-dataset.html) announcing the Speech Commands Dataset. Note that much of what is provided as part of the training set is already public. However, the test set is not.
+- [TensorFlow Audio Recognition Tutorial](https://www.tensorflow.org/versions/master/tutorials/audio_recognition)
+- Link to purchase [Raspberry Pi 3](https://www.amazon.com/CanaKit-Raspberry-Complete-Starter-Kit/dp/B06XW6VX1H) on Amazon. This will be at your own expense.
+- Also review the Prizes tab for details and tools for how the special prize will be evaluated.
+
+## Citation
+
+inversion, Julia Elliott, Mark McDonald, Pete Warden, Raziel. (2017). TensorFlow Speech Recognition Challenge. Kaggle. https://kaggle.com/competitions/tensorflow-speech-recognition-challenge
+
+# Dataset Description
+
+The following is a high level overview of the data for this competition. It is highly recommended that you read the supplementary materials found on the tabs of the [Overview](https://www.kaggle.com/c/tensorflow-speech-recognition-challenge) page.
+
+## File descriptions
+
+- **train.7z** - Contains a few informational files and a folder of audio files. The audio folder contains subfolders with 1 second clips of voice commands, with the folder name being the label of the audio clip. There are more labels that should be predicted. The labels you will need to predict in Test are `yes`, `no`, `up`, `down`, `left`, `right`, `on`, `off`, `stop`, `go`. Everything else should be considered either `unknown` or `silence`. The folder `_background_noise_` contains longer clips of "silence" that you can break up and use as training input.
+    
+    The files contained in the training audio are not uniquely named across labels, but they are unique if you include the label folder. For example, `00f0204f_nohash_0.wav` is found in 14 folders, but that file is a different speech command in each folder.
+    
+    The files are named so the first element is the subject id of the person who gave the voice command, and the last element indicated repeated commands. Repeated commands are when the subject repeats the same word multiple times. Subject id is not provided for the test data, and you can assume that the majority of commands in the test data were from subjects not seen in train.
+    
+    You can expect some inconsistencies in the properties of the training data (e.g., length of the audio).
+    
+- **test.7z** - Contains an audio folder with 150,000+ files in the format `clip_000044442.wav`. The task is to predict the correct label. Not all of the files are evaluated for the leaderboard score.
+- **sample_submission.csv** - A sample submission file in the correct format.
+- **link_to_gcp_credits_form.txt** - Provides the URL to request $500 in GCP credits, provided to the first 500 requestors. Please see [this clarification on credit qualification](https://www.kaggle.com/c/tensorflow-speech-recognition-challenge/discussion/44449).
+
+The following is the README contained in the Train folder, and contains more detailed information.
+
+## Speech Commands Data Set v0.01
+
+This is a set of one-second .wav audio files, each containing a single spoken
+
+English word. These words are from a small set of commands, and are spoken by a
+
+variety of different speakers. The audio files are organized into folders based
+
+on the word they contain, and this data set is designed to help train simple
+
+machine learning models.
+
+It's licensed under the [Creative Commons BY 4.0 license](https://github.com/petewarden/extract_loudest_section). See the LICENSE
+
+file in this folder for full details. Its original location was at
+
+http://download.tensorflow.org/data/speech_commands_v0.01.tar.gz.
+
+## History
+
+This is version 0.01 of the data set containing 64,727 audio files, released on
+August 3rd 2017.
+
+## Collection
+
+The audio files were collected using crowdsourcing, see
+
+[aiyprojects.withgoogle.com/open_speech_recording](https://github.com/petewarden/extract_loudest_section)
+
+for some of the open source audio collection code we used (and please consider
+
+contributing to enlarge this data set). The goal was to gather examples of
+
+people speaking single-word commands, rather than conversational sentences, so
+
+they were prompted for individual words over the course of a five minute
+
+session. Twenty core command words were recorded, with most speakers saying each
+
+of them five times. The core words are "Yes", "No", "Up", "Down", "Left",
+
+"Right", "On", "Off", "Stop", "Go", "Zero", "One", "Two", "Three", "Four",
+
+"Five", "Six", "Seven", "Eight", and "Nine". To help distinguish unrecognized
+
+words, there are also ten auxiliary words, which most speakers only said once.
+
+These include "Bed", "Bird", "Cat", "Dog", "Happy", "House", "Marvin", "Sheila",
+
+## Organization
+
+The files are organized into folders, with each directory name labelling the
+
+word that is spoken in all the contained audio files. No details were kept of
+
+any of the participants age, gender, or location, and random ids were assigned
+
+to each individual. These ids are stable though, and encoded in each file name
+
+as the first part before the underscore. If a participant contributed multiple
+
+utterances of the same word, these are distinguished by the number at the end of
+
+the file name. For example, the file path `happy/3cfc6b3a_nohash_2.wav`
+
+indicates that the word spoken was "happy", the speaker's id was "3cfc6b3a", and
+
+this is the third utterance of that word by this speaker in the data set. The
+
+'nohash' section is to ensure that all the utterances by a single speaker are
+
+sorted into the same training partition, to keep very similar repetitions from
+
+giving unrealistically optimistic evaluation scores.
+
+## Partitioning
+
+The audio clips haven't been separated into training, test, and validation sets
+
+explicitly, but by convention a hashing function is used to stably assign each
+
+file to a set. Here's some Python code demonstrating how a complete file path
+
+and the desired validation and test set sizes (usually both 10%) are used to
+
+assign a set:
+
+```python
+MAX_NUM_WAVS_PER_CLASS = 2**27 - 1  # ~134M
+
+def which_set(filename, validation_percentage, testing_percentage):
+  """Determines which data partition the file should belong to.
+
+  We want to keep files in the same training, validation, or testing sets even
+  if new ones are added over time. This makes it less likely that testing
+  samples will accidentally be reused in training when long runs are restarted
+  for example. To keep this stability, a hash of the filename is taken and used
+  to determine which set it should belong to. This determination only depends on
+  the name and the set proportions, so it won't change as other files are added.
+
+  It's also useful to associate particular files as related (for example words
+  spoken by the same person), so anything after '_nohash_' in a filename is
+  ignored for set determination. This ensures that 'bobby_nohash_0.wav' and
+  'bobby_nohash_1.wav' are always in the same set, for example.
+
+  Args:
+    filename: File path of the data sample.
+    validation_percentage: How much of the data set to use for validation.
+    testing_percentage: How much of the data set to use for testing.
+
+  Returns:
+    String, one of 'training', 'validation', or 'testing'.
+  """
+  base_name = os.path.basename(filename)
+  # We want to ignore anything after '_nohash_' in the file name when
+  # deciding which set to put a wav in, so the data set creator has a way of
+  # grouping wavs that are close variations of each other.
+  hash_name = re.sub(r'_nohash_.*$', '', base_name)
+  # This looks a bit magical, but we need to decide whether this file should
+  # go into the training, testing, or validation sets, and we want to keep
+  # existing files in the same set even if more files are subsequently
+  # added.
+  # To do that, we need a stable way of deciding based on just the file name
+  # itself, so we do a hash of that and then use that to generate a
+  # probability value that we use to assign it.
+  hash_name_hashed = hashlib.sha1(hash_name).hexdigest()
+  percentage_hash = ((int(hash_name_hashed, 16) %
+                      (MAX_NUM_WAVS_PER_CLASS + 1)) *
+                    (100.0 / MAX_NUM_WAVS_PER_CLASS))
+  if percentage_hash < validation_percentage:
+    result = 'validation'
+  elif percentage_hash < (testing_percentage + validation_percentage):
+    result = 'testing'
+  else:
+    result = 'training'
+  return result
+```
+
+The results of running this over the current set are included in this archive as
+
+validation_list.txt and testing_list.txt. These text files contain the paths to
+
+all the files in each set, with each path on a new line. Any files that aren't
+
+in either of these lists can be considered to be part of the training set.
+
+## Processing
+
+The original audio files were collected in uncontrolled locations by people
+
+around the world. We requested that they do the recording in a closed room for
+
+privacy reasons, but didn't stipulate any quality requirements. This was by
+
+design, since we wanted examples of the sort of speech data that we're likely to
+
+encounter in consumer and robotics applications, where we don't have much
+
+control over the recording equipment or environment. The data was captured in a
+
+variety of formats, for example Ogg Vorbis encoding for the web app, and then
+
+converted to a 16-bit little-endian PCM-encoded WAVE file at a 16000 sample
+
+rate. The audio was then trimmed to a one second length to align most
+
+utterances, using the
+
+[extract_loudest_section](https://github.com/petewarden/extract_loudest_section)
+
+tool. The audio files were then screened for silence or incorrect words, and
+
+arranged into folders by label.
+
+## Background Noise
+
+To help train networks to cope with noisy environments, it can be helpful to mix
+
+in realistic background audio. The `_background_noise_` folder contains a set of
+
+longer audio clips that are either recordings or mathematical simulations of
+
+noise. For more details, see the `_background_noise_/README.md`.
+
+## Citations
+
+If you use the Speech Commands dataset in your work, please cite it as:
+
+APA-style citation: "Warden P. Speech Commands: A public dataset for single-word
+
+speech recognition, 2017. Available from
+
+http://download.tensorflow.org/data/speech_commands_v0.01.tar.gz".
+
+BibTeX `@article{speechcommands, title={Speech Commands: A public dataset for single-word speech recognition.}, author={Warden, Pete}, journal={Dataset available from http://download.tensorflow.org/data/speech_commands_v0.01.tar.gz}, year={2017} }`
+
+## Credits
+
+Massive thanks are due to everyone who donated recordings to this data set, I'm
+
+very grateful. I also couldn't have put this together without the help and
+
+support of Billy Rutledge, Rajat Monga, Raziel Alvarez, Brad Krueger, Barbara
+
+Petit, Gursheesh Kour, and all the AIY and TensorFlow teams.
+
+Pete Warden, [petewarden@google.com](mailto:petewarden@google.com)

dslighting/registry/tensorflow-speech-recognition-challenge/description_obfuscated.md

@@ -0,0 +1,36 @@
+# Task
+
+Build an algorithm that understands simple spoken commands.
+
+# Metric
+
+Multiclass Accuracy.
+
+There are 12 possible labels for the Test set: `yes`, `no`, `up`, `down`, `left`, `right`, `on`, `off`, `stop`, `go`, `silence`, `unknown`.
+
+The `unknown` label should be used for a command that is not one one of the first 10 labels or that is not `silence`.
+
+# Submission Format
+
+For audio clip in the test set, you must predict the correct `label`. The submission file should contain a header and have the following format:
+
+```
+fname,label
+clip_000044442.wav,silence
+clip_0000adecb.wav,left
+clip_0000d4322.wav,unknown
+etc.
+```
+
+# Dataset
+
+- **train.7z** - Contains a few informational files and a folder of audio files. The audio folder contains subfolders with 1 second clips of voice commands, with the folder name being the label of the audio clip. There are more labels that should be predicted. The labels you will need to predict in Test are `yes`, `no`, `up`, `down`, `left`, `right`, `on`, `off`, `stop`, `go`. Everything else should be considered either `unknown` or `silence`. The folder `_background_noise_` contains longer clips of "silence" that you can break up and use as training input.
+    
+    The files contained in the training audio are not uniquely named across labels, but they are unique if you include the label folder. For example, `00f0204f_nohash_0.wav` is found in 14 folders, but that file is a different speech command in each folder.
+    
+    The files are named so the first element is the subject id of the person who gave the voice command, and the last element indicated repeated commands. Repeated commands are when the subject repeats the same word multiple times. Subject id is not provided for the test data, and you can assume that the majority of commands in the test data were from subjects not seen in train.
+    
+    You can expect some inconsistencies in the properties of the training data (e.g., length of the audio).
+    
+- **test.7z** - Contains an audio folder with 150,000+ files in the format `clip_000044442.wav`. The task is to predict the correct label. Not all of the files are evaluated for the leaderboard score.
+- **sample_submission.csv** - A sample submission file in the correct format.

dslighting/registry/tensorflow-speech-recognition-challenge/grade.py

@@ -0,0 +1,11 @@
+import pandas as pd
+from sklearn.metrics import accuracy_score
+
+from mlebench.competitions.utils import prepare_for_accuracy_metric
+
+
+def grade(submission: pd.DataFrame, answers: pd.DataFrame) -> float:
+    accuracy_inputs = prepare_for_accuracy_metric(
+        submission=submission, answers=answers, target_column="label", id_column="fname"
+    )
+    return accuracy_score(**accuracy_inputs)

dslighting/registry/tensorflow-speech-recognition-challenge/prepare.py

@@ -0,0 +1,90 @@
+import shutil
+from dataclasses import dataclass
+from pathlib import Path
+
+import pandas as pd
+from sklearn.model_selection import train_test_split
+from tqdm import tqdm
+
+from mlebench.utils import extract
+
+CLASSES = [
+    "yes",
+    "no",
+    "up",
+    "down",
+    "left",
+    "right",
+    "on",
+    "off",
+    "stop",
+    "go",
+    "unknown",
+    "silence",
+]
+
+
+@dataclass(frozen=True)
+class AudioFile:
+    label: str
+    path: Path
+
+
+def prepare(raw: Path, public: Path, private: Path):
+    # extract only what we need
+    extract(raw / "train.7z", raw)
+
+    # Create train, test from train split
+    audio_dir = raw / "train" / "audio"
+    audio_files = sorted(
+        [AudioFile(fpath.parent.name, fpath) for fpath in audio_dir.rglob("*.wav")],
+        key=lambda x: f"{x.label}_{x.path.name}",
+    )
+    train_files, test_files = train_test_split(audio_files, test_size=0.1, random_state=0)
+
+    # Make necessary directories
+    labels = list(
+        dict.fromkeys([file.label for file in train_files])
+    )  # Gets unique elements deterministically
+
+    for label in labels:
+        (public / "train" / "audio" / label).mkdir(parents=True, exist_ok=True)
+
+    (public / "test" / "audio").mkdir(parents=True, exist_ok=True)
+
+    # Copy over train and test files
+    for file in tqdm(train_files, desc="Copying train files"):
+        shutil.copyfile(
+            src=file.path,
+            dst=public / "train" / "audio" / file.label / file.path.name,
+        )
+
+    test_records = []
+
+    for idx, file in enumerate(tqdm(test_files, desc="Copying test files")):
+        # Rename files, since training audio files across labels aren't necessarily unique.
+        new_id = str(idx).zfill(8)
+        new_name = f"clip_{new_id}.wav"
+        test_records.append({"fname": new_name, "label": file.label})
+
+        shutil.copyfile(
+            src=file.path,
+            dst=public / "test" / "audio" / new_name,
+        )
+
+    test = pd.DataFrame.from_records(test_records)
+    test.to_csv(private / "test.csv", index=False)
+
+    test_without_labels = test.drop(columns=["label"])
+    sample_submission = test_without_labels.copy()
+    sample_submission["label"] = "silence"
+    sample_submission.to_csv(public / "sample_submission.csv", index=False)
+
+    # Sanity checks
+    test_audio_files = list((public / "test" / "audio").glob("*.wav"))
+    num_test_files = len(test_audio_files)
+    num_submission_entries = len(sample_submission)
+    assert num_test_files == num_submission_entries, (
+        f"The number of test audio files ({num_test_files}) does not match the number of entries "
+        f"in sample_submission.csv ({num_submission_entries}). Please check the file copying process."
+    )