dslighting 1.7.1__py3-none-any.whl → 1.7.8__py3-none-any.whl

mlebench/competitions/tabular-playground-series-may-2022/prepare.py

@@ -0,0 +1,56 @@
+from pathlib import Path
+
+import numpy as np
+from sklearn.model_selection import train_test_split
+
+from mlebench.utils import read_csv
+
+
+def prepare(raw: Path, public: Path, private: Path):
+
+    old_train = read_csv(raw / "train.csv")
+
+    # 900k train, 1.6m - 900k = 700k test; so 700k/1.6m = 0.4375
+    # We create our split at 100,000 test samples to get same OOM while keeping as many samples as possible in train
+    new_train, new_test = train_test_split(old_train, test_size=100_000, random_state=0)
+
+    # make ids go from 0 to len(new_train) - 1
+    new_train.id = np.arange(len(new_train))
+    # and from len(new_train) to len(new_train) + len(new_test) - 1
+    new_test.id = np.arange(len(new_train), len(new_train) + len(new_test))
+
+    # make downstream files
+    new_test_without_labels = new_test.drop(columns=["target"]).copy()
+    gold_submission = new_test[["id", "target"]].copy()
+    sample_submission = gold_submission.copy()
+    sample_submission.target = 0.5
+
+    # save
+    new_train.to_csv(public / "train.csv", index=False)
+    new_test.to_csv(private / "test.csv", index=False)
+    new_test_without_labels.to_csv(public / "test.csv", index=False)
+    gold_submission.to_csv(private / "gold_submission.csv", index=False)
+    sample_submission.to_csv(public / "sample_submission.csv", index=False)
+
+    # checks
+    assert len(new_train) + len(new_test) == len(
+        old_train
+    ), "Expected the sum of the lengths of the new train and test to be equal to the length of the original train."
+    assert len(new_test) == len(
+        sample_submission
+    ), "Expected the length of the private test to be equal to the length of the sample submission."
+    assert len(new_test) == len(
+        gold_submission
+    ), "Expected the length of the private test to be equal to the length of the gold submission."
+
+    assert (
+        new_train.columns.to_list() == old_train.columns.to_list()
+    ), "Expected the columns of the new train to be the same as the columns of the original train."
+    assert (
+        new_test.columns.to_list() == old_train.columns.to_list()
+    ), "Expected the columns of the new test to be the same as the columns of the original train"
+
+    # check that ids dont overlap between train and test
+    assert set(new_train.id).isdisjoint(
+        set(new_test.id)
+    ), "Expected the ids of the new train and test to be disjoint."

mlebench/competitions/tabular-playground-series-may-2022/prepare_val.py

@@ -0,0 +1,116 @@
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+from sklearn.model_selection import train_test_split
+
+from mlebench.utils import read_csv
+
+
+def _create_and_save_split(
+    df_to_split: pd.DataFrame,
+    public_path: Path,
+    private_path: Path,
+    test_size: int,
+    random_state: int,
+):
+    """
+    Helper function to perform a data split, re-index, and save all required files.
+
+    Args:
+        df_to_split (pd.DataFrame): The dataframe to be split.
+        public_path (Path): The directory for public-facing files (train, test features).
+        private_path (Path): The directory for private-facing files (test labels).
+        test_size (int): The number of samples for the test set.
+        random_state (int): The random state for reproducibility.
+
+    Returns:
+        pd.DataFrame: The new, smaller training dataframe from the split.
+    """
+    # Create output directories if they don't exist
+    public_path.mkdir(parents=True, exist_ok=True)
+    private_path.mkdir(parents=True, exist_ok=True)
+
+    # Perform the split
+    train_df, test_df = train_test_split(
+        df_to_split, test_size=test_size, random_state=random_state
+    )
+
+    # make ids go from 0 to len(train_df) - 1
+    train_df.id = np.arange(len(train_df))
+    # and from len(train_df) to len(train_df) + len(test_df) - 1
+    test_df.id = np.arange(len(train_df), len(train_df) + len(test_df))
+
+    # make downstream files
+    test_df_without_labels = test_df.drop(columns=["target"]).copy()
+    gold_submission = test_df[["id", "target"]].copy()
+    sample_submission = gold_submission.copy()
+    sample_submission.target = 0.5
+
+    # save files with required names
+    train_df.to_csv(public_path / "train.csv", index=False)
+    test_df.to_csv(private_path / "test.csv", index=False)
+    test_df_without_labels.to_csv(public_path / "test.csv", index=False)
+    gold_submission.to_csv(private_path / "gold_submission.csv", index=False)
+    sample_submission.to_csv(public_path / "sample_submission.csv", index=False)
+
+    # run checks for this split
+    assert len(train_df) + len(test_df) == len(
+        df_to_split
+    ), "Expected the sum of the lengths of the new train and test to be equal to the length of the input data."
+    assert len(test_df) == len(
+        sample_submission
+    ), "Expected the length of the private test to be equal to the length of the sample submission."
+    assert len(test_df) == len(
+        gold_submission
+    ), "Expected the length of the private test to be equal to the length of the gold submission."
+    assert (
+        train_df.columns.to_list() == df_to_split.columns.to_list()
+    ), "Expected the columns of the new train to be the same as the columns of the input data."
+    assert (
+        test_df.columns.to_list() == df_to_split.columns.to_list()
+    ), "Expected the columns of the new test to be the same as the columns of the input data"
+    assert set(train_df.id).isdisjoint(
+        set(test_df.id)
+    ), "Expected the ids of the new train and test to be disjoint."
+
+    return train_df
+
+
+def prepare(raw: Path, public: Path, private: Path):
+
+    old_train = read_csv(raw / "train.csv")
+
+    # --- Stage 1: Create the original train/test split ---
+    # This section produces the primary competition data. Its outputs in `public/`
+    # and `private/` are guaranteed to be identical to the original script.
+
+    # 900k train, 1.6m - 900k = 700k test; so 700k/1.6m = 0.4375
+    # We create our split at 100,000 test samples to get same OOM while keeping as many samples as possible in train
+    new_train = _create_and_save_split(
+        df_to_split=old_train,
+        public_path=public,
+        private_path=private,
+        test_size=100_000,
+        random_state=0,
+    )
+
+    # --- Stage 2: Create the new train/validation split ---
+    # This section takes the `new_train` set from Stage 1 and splits it again
+    # to create a smaller training set and a new validation set.
+    # The outputs are saved to parallel `public_val/` and `private_val/` directories.
+
+    # Define paths for the new validation set directories, parallel to the original ones.
+    public_val = public.parent / "public_val"
+    private_val = private.parent / "private_val"
+
+    # Split `new_train` again using the exact same logic and test size.
+    # The resulting `test_val` set will have 100,000 samples, same as the original `test` set.
+    # The filenames and directory structure are mirrored for consistency.
+    _create_and_save_split(
+        df_to_split=new_train,
+        public_path=public_val,
+        private_path=private_val,
+        test_size=100_000,
+        random_state=0,
+    )

mlebench/competitions/tensorflow-speech-recognition-challenge/grade.py

@@ -0,0 +1,11 @@
+import pandas as pd
+from sklearn.metrics import accuracy_score
+
+from mlebench.competitions.utils import prepare_for_accuracy_metric
+
+
+def grade(submission: pd.DataFrame, answers: pd.DataFrame) -> float:
+    accuracy_inputs = prepare_for_accuracy_metric(
+        submission=submission, answers=answers, target_column="label", id_column="fname"
+    )
+    return accuracy_score(**accuracy_inputs)

mlebench/competitions/tensorflow-speech-recognition-challenge/prepare.py

@@ -0,0 +1,90 @@
+import shutil
+from dataclasses import dataclass
+from pathlib import Path
+
+import pandas as pd
+from sklearn.model_selection import train_test_split
+from tqdm import tqdm
+
+from mlebench.utils import extract
+
+CLASSES = [
+    "yes",
+    "no",
+    "up",
+    "down",
+    "left",
+    "right",
+    "on",
+    "off",
+    "stop",
+    "go",
+    "unknown",
+    "silence",
+]
+
+
+@dataclass(frozen=True)
+class AudioFile:
+    label: str
+    path: Path
+
+
+def prepare(raw: Path, public: Path, private: Path):
+    # extract only what we need
+    extract(raw / "train.7z", raw)
+
+    # Create train, test from train split
+    audio_dir = raw / "train" / "audio"
+    audio_files = sorted(
+        [AudioFile(fpath.parent.name, fpath) for fpath in audio_dir.rglob("*.wav")],
+        key=lambda x: f"{x.label}_{x.path.name}",
+    )
+    train_files, test_files = train_test_split(audio_files, test_size=0.1, random_state=0)
+
+    # Make necessary directories
+    labels = list(
+        dict.fromkeys([file.label for file in train_files])
+    )  # Gets unique elements deterministically
+
+    for label in labels:
+        (public / "train" / "audio" / label).mkdir(parents=True, exist_ok=True)
+
+    (public / "test" / "audio").mkdir(parents=True, exist_ok=True)
+
+    # Copy over train and test files
+    for file in tqdm(train_files, desc="Copying train files"):
+        shutil.copyfile(
+            src=file.path,
+            dst=public / "train" / "audio" / file.label / file.path.name,
+        )
+
+    test_records = []
+
+    for idx, file in enumerate(tqdm(test_files, desc="Copying test files")):
+        # Rename files, since training audio files across labels aren't necessarily unique.
+        new_id = str(idx).zfill(8)
+        new_name = f"clip_{new_id}.wav"
+        test_records.append({"fname": new_name, "label": file.label})
+
+        shutil.copyfile(
+            src=file.path,
+            dst=public / "test" / "audio" / new_name,
+        )
+
+    test = pd.DataFrame.from_records(test_records)
+    test.to_csv(private / "test.csv", index=False)
+
+    test_without_labels = test.drop(columns=["label"])
+    sample_submission = test_without_labels.copy()
+    sample_submission["label"] = "silence"
+    sample_submission.to_csv(public / "sample_submission.csv", index=False)
+
+    # Sanity checks
+    test_audio_files = list((public / "test" / "audio").glob("*.wav"))
+    num_test_files = len(test_audio_files)
+    num_submission_entries = len(sample_submission)
+    assert num_test_files == num_submission_entries, (
+        f"The number of test audio files ({num_test_files}) does not match the number of entries "
+        f"in sample_submission.csv ({num_submission_entries}). Please check the file copying process."
+    )

mlebench/competitions/tensorflow-speech-recognition-challenge/prepare_val.py

@@ -0,0 +1,148 @@
+import shutil
+from dataclasses import dataclass
+from pathlib import Path
+from typing import List
+
+import pandas as pd
+from sklearn.model_selection import train_test_split
+from tqdm import tqdm
+
+from mlebench.utils import extract
+
+CLASSES = [
+    "yes",
+    "no",
+    "up",
+    "down",
+    "left",
+    "right",
+    "on",
+    "off",
+    "stop",
+    "go",
+    "unknown",
+    "silence",
+]
+
+
+@dataclass(frozen=True)
+class AudioFile:
+    label: str
+    path: Path
+
+
+def _create_and_populate_split(
+    files_to_split: List[AudioFile],
+    public_dir: Path,
+    private_dir: Path,
+    test_size: float,
+    random_state: int,
+) -> List[AudioFile]:
+    """
+    Splits a list of audio files into train and test sets, and populates the
+    corresponding public and private directories.
+
+    Args:
+        files_to_split: A list of AudioFile objects to be split.
+        public_dir: The public directory to save training data and test stubs.
+        private_dir: The private directory to save test ground truth.
+        test_size: The proportion of the dataset to allocate to the test split.
+        random_state: The seed used by the random number generator.
+
+    Returns:
+        A list of AudioFile objects that were assigned to the training set.
+    """
+    train_files, test_files = train_test_split(
+        files_to_split, test_size=test_size, random_state=random_state
+    )
+
+    # Make necessary directories
+    public_dir.mkdir(parents=True, exist_ok=True)
+    private_dir.mkdir(parents=True, exist_ok=True)
+
+    labels = list(
+        dict.fromkeys([file.label for file in train_files])
+    )  # Gets unique elements deterministically
+
+    for label in labels:
+        (public_dir / "train" / "audio" / label).mkdir(parents=True, exist_ok=True)
+
+    (public_dir / "test" / "audio").mkdir(parents=True, exist_ok=True)
+
+    # Copy over train and test files
+    desc_suffix = public_dir.name
+    for file in tqdm(train_files, desc=f"Copying train files to {desc_suffix}"):
+        shutil.copyfile(
+            src=file.path,
+            dst=public_dir / "train" / "audio" / file.label / file.path.name,
+        )
+
+    test_records = []
+
+    for idx, file in enumerate(tqdm(test_files, desc=f"Copying test files to {desc_suffix}")):
+        # Rename files, since training audio files across labels aren't necessarily unique.
+        new_id = str(idx).zfill(8)
+        new_name = f"clip_{new_id}.wav"
+        test_records.append({"fname": new_name, "label": file.label})
+
+        shutil.copyfile(
+            src=file.path,
+            dst=public_dir / "test" / "audio" / new_name,
+        )
+
+    test = pd.DataFrame.from_records(test_records)
+    test.to_csv(private_dir / "test.csv", index=False)
+
+    test_without_labels = test.drop(columns=["label"])
+    sample_submission = test_without_labels.copy()
+    sample_submission["label"] = "silence"
+    sample_submission.to_csv(public_dir / "sample_submission.csv", index=False)
+
+    # Sanity checks
+    test_audio_files = list((public_dir / "test" / "audio").glob("*.wav"))
+    num_test_files = len(test_audio_files)
+    num_submission_entries = len(sample_submission)
+    assert num_test_files == num_submission_entries, (
+        f"The number of test audio files ({num_test_files}) does not match the number of entries "
+        f"in sample_submission.csv ({num_submission_entries}). Please check the file copying process."
+    )
+
+    return train_files
+
+
+def prepare(raw: Path, public: Path, private: Path):
+    # extract only what we need
+    extract(raw / "train.7z", raw)
+
+    # Create train, test from train split
+    audio_dir = raw / "train" / "audio"
+    all_audio_files = sorted(
+        [AudioFile(fpath.parent.name, fpath) for fpath in audio_dir.rglob("*.wav")],
+        key=lambda x: f"{x.label}_{x.path.name}",
+    )
+
+    # --- First Split: Create the original train/test sets ---
+    # This call populates the original `public` and `private` directories.
+    # It returns the files assigned to the training set for the next split.
+    train_files_for_val_split = _create_and_populate_split(
+        files_to_split=all_audio_files,
+        public_dir=public,
+        private_dir=private,
+        test_size=0.1,
+        random_state=0,
+    )
+
+    # --- Second Split: Create the new train_val/test_val sets ---
+    # Define new paths for the validation split, parallel to the original ones.
+    public_val = public.parent / "public_val"
+    private_val = private.parent / "private_val"
+
+    # To get a validation test set of the same size as the original test set,
+    # we take a 1/9 slice of the remaining 90% of the data (0.1 / 0.9 = 1/9).
+    _create_and_populate_split(
+        files_to_split=train_files_for_val_split,
+        public_dir=public_val,
+        private_dir=private_val,
+        test_size=1 / 9,
+        random_state=0, # Use the same random_state for deterministic behavior
+    )

mlebench/competitions/tensorflow2-question-answering/grade.py

@@ -0,0 +1,122 @@
+import pandas as pd
+from sklearn.metrics import f1_score
+
+from mlebench.grade_helpers import InvalidSubmissionError
+
+
+def prepare_for_metric(submission: pd.DataFrame, answers: dict):
+    """
+    `submission` is a pd.DataFrame with "example_id" and "PredictionString" columns.
+
+    `answers` is a dict in the format of "simplified-nq-train.jsonl" described here:
+    https://www.kaggle.com/c/tensorflow2-question-answering/data
+
+    The competition uses a micro F1 score metric, which is a binary classification metric.
+    For retrieval, we convert the submission and answers for each sample as:
+    - y_pred: if the submission has an answer, it is 1, else 0
+    - y_true:
+        - if the submission has an answer and it exists in the true labels, y_true=1
+        - if the submission has an answer but it does not exist in the true labels, y_true=0
+        - if the submission has no answer but there exists a true label, y_true=1
+        - if the submission has no answer and there is no true label, y_true=0
+
+    This is consistent with
+    https://www.kaggle.com/competitions/tensorflow2-question-answering/overview/evaluation
+    - TP = the predicted indices match one of the possible ground truth indices
+    - FP = the predicted indices do NOT match one of the possible ground truth indices, OR
+        a prediction has been made where no ground truth exists
+    - FN = no prediction has been made where a ground truth exists
+
+    Returns y_true, y_pred which are lists of 0s and 1s.
+    """
+    if len(submission) != 2 * len(answers):
+        raise InvalidSubmissionError(
+            f"Submission length {len(submission)} != 2 * answers length {len(answers)}"
+        )
+    # Empty strings are read as NaN by pandas, but we want these to remain empty strings
+    submission.fillna("", inplace=True)
+    submission = submission.astype(str)
+
+    expected_ids = []
+    for sample in answers:
+        expected_ids.append(f"{sample['example_id']}_long")
+        expected_ids.append(f"{sample['example_id']}_short")
+
+    if not ({"example_id", "PredictionString"}).issubset(submission.columns):
+        raise InvalidSubmissionError(
+            "Submission requires 'example_id' and 'PredictionString' columns."
+        )
+    if not set(submission["example_id"]) == set(expected_ids):
+        raise InvalidSubmissionError(
+            "Submission example_ids do not match expected_ids. Please ensure you have a "
+            "long and short answer for each example_id in the answers."
+        )
+    if not len(set(submission["example_id"])) == len(submission):
+        raise InvalidSubmissionError(
+            "Submission example_ids are not unique. Please ensure you have a "
+            "long and short answer for each example_id in the answers."
+        )
+
+    y_pred = [1 if row["PredictionString"].strip() else 0 for _, row in submission.iterrows()]
+
+    y_true = []
+    true_annotations = {sample["example_id"]: sample["annotations"][0] for sample in answers}
+    for idx, row in submission.iterrows():
+        sample_id = row["example_id"]
+
+        # Parse prediction
+        # Prediction may be any of "{start_token}:{end_token}", "YES", "NO", ""
+        pred = row["PredictionString"].strip()
+        if ":" in pred and len(pred.split(":")) == 2:
+            start_token, end_token = pred.split(":")
+            pred = (int(start_token), int(end_token))
+        elif pred in ["YES", "NO", ""]:
+            pass
+        else:
+            raise InvalidSubmissionError(f"Invalid submission format: {pred}")
+
+        # Parse valid answers
+        if sample_id.endswith("_long"):
+            sample_id = sample_id.split("_long")[0]
+            annotation = true_annotations[sample_id]
+            valid_answers = []
+            if annotation["long_answer"]["start_token"] != -1:
+                valid_answers.append(
+                    (
+                        int(annotation["long_answer"]["start_token"]),
+                        int(annotation["long_answer"]["end_token"]),
+                    ),
+                )
+        elif sample_id.endswith("_short"):
+            sample_id = sample_id.split("_short")[0]
+            annotation = true_annotations[sample_id]
+            valid_answers = [
+                (int(short_answer["start_token"]), int(short_answer["end_token"]))
+                for short_answer in annotation["short_answers"]
+            ]
+            if annotation["yes_no_answer"] != "NONE":
+                valid_answers.append(annotation["yes_no_answer"])
+        else:
+            raise InvalidSubmissionError(
+                f"sample_id {sample_id} does not end with '_long' or '_short'"
+            )
+
+        # Mark as 0 or 1 based on the above rules
+        if pred:
+            if pred in valid_answers:
+                y_true.append(1)
+            else:
+                y_true.append(0)
+        else:
+            if valid_answers:
+                y_true.append(1)
+            else:
+                y_true.append(0)
+
+    assert len(y_true) == len(y_pred), f"y_true {len(y_true)} != y_pred {len(y_pred)}"
+    return y_true, y_pred
+
+
+def grade(submission: pd.DataFrame, answers: dict) -> float:
+    y_true, y_pred = prepare_for_metric(submission, answers)
+    return f1_score(y_true=y_true, y_pred=y_pred, average="micro")

mlebench/competitions/tensorflow2-question-answering/prepare.py

@@ -0,0 +1,122 @@
+from pathlib import Path
+
+import pandas as pd
+from sklearn.model_selection import train_test_split
+from tqdm import tqdm
+
+from mlebench.utils import get_logger
+
+logger = get_logger(__name__)
+
+
+def extract_string(document_text: str, start_token_idx: int, end_token_idx: int) -> str:
+    document_tokens = document_text.split(" ")
+    extract_tokens = document_tokens[start_token_idx:end_token_idx]
+    return " ".join(extract_tokens)
+
+
+def prepare(raw: Path, public: Path, private: Path):
+    """
+    Splits the data in raw into public and private datasets with appropriate test/train splits.
+    """
+
+    # Create train, test from train split
+    train_file = "simplified-nq-train.jsonl"
+
+    logger.info("Counting lines in train file...")
+    with open(raw / train_file, "r") as f:
+        n_lines = sum(1 for _ in f)
+    logger.info(f"Found {n_lines} lines in train file.")
+
+    # Read data in chunks to avoid memory issues
+    train_ids, test_ids = [], []
+    lightweight_test = []  # We'll use this to create a gold submission later
+    with tqdm(total=n_lines, desc="Splitting data") as pbar:
+        for df in pd.read_json(raw / train_file, orient="records", lines=True, chunksize=1_000):
+            # Convert IDs to strings, Kaggle.com is inconsistent about this but strings make more sense
+            df["example_id"] = df["example_id"].astype(str)
+            new_train, new_test = train_test_split(df, test_size=0.1, random_state=0)
+
+            keys_to_keep = [
+                "example_id",
+                "question_text",
+                "document_text",
+                "long_answer_candidates",
+            ]
+            new_test_without_labels = new_test.copy()[keys_to_keep]
+
+            # Append lines to new train and test
+            with open(public / "simplified-nq-train.jsonl", "a") as f:
+                f.write(new_train.to_json(orient="records", lines=True))
+            with open(private / "test.jsonl", "a") as f:
+                f.write(new_test.to_json(orient="records", lines=True))
+            with open(public / "simplified-nq-test.jsonl", "a") as f:
+                f.write(new_test_without_labels.to_json(orient="records", lines=True))
+
+            train_ids.extend(new_train["example_id"].tolist())
+            test_ids.extend(new_test["example_id"].tolist())
+            lightweight_test.append(
+                new_test.copy()[["example_id", "question_text", "annotations"]]
+            )  # For gold submission
+            pbar.update(len(df))
+
+    lightweight_test = pd.concat(lightweight_test, ignore_index=True)
+
+    assert len(train_ids) + len(test_ids) == n_lines
+    assert len(lightweight_test) == len(test_ids)
+
+    # Create a gold submission with columns "example_id", "PredictionString"
+    gold_rows = []
+    for idx, sample in tqdm(
+        lightweight_test.iterrows(), total=len(lightweight_test), desc="Creating gold submission"
+    ):
+        sample = sample.to_dict()
+        assert len(sample["annotations"]) == 1
+        annotation = sample["annotations"][0]
+
+        # Create short answer
+
+        # Multiple answers are possible: yes_no_answer or one of short_answers
+        # We just take the first one
+        if annotation["yes_no_answer"] != "NONE":
+            answer = annotation["yes_no_answer"]
+        elif len(annotation["short_answers"]) > 0:
+            start_token = annotation["short_answers"][0]["start_token"]
+            end_token = annotation["short_answers"][0]["end_token"]
+            answer = f"{start_token}:{end_token}"
+        else:
+            answer = ""
+
+        logger.debug(f"q: {sample['question_text']}")
+        logger.debug(f"a: {answer}")
+        logger.debug("")
+
+        gold_rows.append(
+            {"example_id": f"{sample['example_id']}_short", "PredictionString": answer}
+        )
+
+        # Create long answer
+
+        if annotation["long_answer"]["start_token"] != -1:
+            start_token = annotation["long_answer"]["start_token"]
+            end_token = annotation["long_answer"]["end_token"]
+            answer = f"{start_token}:{end_token}"
+        else:
+            answer = ""
+
+        logger.debug(f"q: {sample['question_text']}")
+        logger.debug(f"a: {answer}")
+        logger.debug("")
+
+        gold_rows.append({"example_id": f"{sample['example_id']}_long", "PredictionString": answer})
+
+    gold_submission = pd.DataFrame(gold_rows)
+    gold_submission.to_csv(private / "gold_submission.csv", index=False)
+
+    # Sample submission
+    sample_submission = gold_submission.copy()
+    sample_submission["PredictionString"] = ""
+    sample_submission.to_csv(public / "sample_submission.csv", index=False)
+
+    assert len(gold_submission) == 2 * len(test_ids)
+    assert len(sample_submission) == 2 * len(test_ids)