dslighting 1.3.9__py3-none-any.whl → 1.7.6__py3-none-any.whl

dslighting/registry/tensorflow2-question-answering/prepare_val.py

@@ -0,0 +1,187 @@
+from pathlib import Path
+
+import pandas as pd
+from sklearn.model_selection import train_test_split
+from tqdm import tqdm
+
+from mlebench.utils import get_logger
+
+logger = get_logger(__name__)
+
+
+def extract_string(document_text: str, start_token_idx: int, end_token_idx: int) -> str:
+    document_tokens = document_text.split(" ")
+    extract_tokens = document_tokens[start_token_idx:end_token_idx]
+    return " ".join(extract_tokens)
+
+
+def create_submission_files(
+    lightweight_test_df: pd.DataFrame, public_path: Path, private_path: Path, n_test_samples: int
+):
+    """
+    Creates the gold submission (private) and sample submission (public) files.
+    """
+    # Create a gold submission with columns "example_id", "PredictionString"
+    gold_rows = []
+    for idx, sample in tqdm(
+        lightweight_test_df.iterrows(),
+        total=len(lightweight_test_df),
+        desc=f"Creating submission files for {public_path.name}",
+    ):
+        sample = sample.to_dict()
+        assert len(sample["annotations"]) == 1
+        annotation = sample["annotations"][0]
+
+        # Create short answer
+        # Multiple answers are possible: yes_no_answer or one of short_answers
+        # We just take the first one
+        if annotation["yes_no_answer"] != "NONE":
+            answer = annotation["yes_no_answer"]
+        elif len(annotation["short_answers"]) > 0:
+            start_token = annotation["short_answers"][0]["start_token"]
+            end_token = annotation["short_answers"][0]["end_token"]
+            answer = f"{start_token}:{end_token}"
+        else:
+            answer = ""
+
+        gold_rows.append(
+            {"example_id": f"{sample['example_id']}_short", "PredictionString": answer}
+        )
+
+        # Create long answer
+        if annotation["long_answer"]["start_token"] != -1:
+            start_token = annotation["long_answer"]["start_token"]
+            end_token = annotation["long_answer"]["end_token"]
+            answer = f"{start_token}:{end_token}"
+        else:
+            answer = ""
+
+        gold_rows.append({"example_id": f"{sample['example_id']}_long", "PredictionString": answer})
+
+    gold_submission = pd.DataFrame(gold_rows)
+    gold_submission.to_csv(private_path / "gold_submission.csv", index=False)
+
+    # Sample submission
+    sample_submission = gold_submission.copy()
+    sample_submission["PredictionString"] = ""
+    sample_submission.to_csv(public_path / "sample_submission.csv", index=False)
+
+    assert len(gold_submission) == 2 * n_test_samples
+    assert len(sample_submission) == 2 * n_test_samples
+
+
+def prepare(raw: Path, public: Path, private: Path):
+    """
+    Splits the data in raw into public and private datasets with appropriate test/train splits.
+    Then, it further splits the created train set into a smaller train set and a validation set.
+    """
+    # Define and create paths for the new validation split
+    public_val = public.parent / "public_val"
+    private_val = private.parent / "private_val"
+    public_val.mkdir(exist_ok=True)
+    private_val.mkdir(exist_ok=True)
+
+    # --- STAGE 1: Create Original Train/Test Split ---
+    # This section creates the original `public` and `private` outputs.
+
+    train_file = "simplified-nq-train.jsonl"
+    public_train_file = public / "simplified-nq-train.jsonl"
+
+    logger.info("Counting lines in raw data file...")
+    with open(raw / train_file, "r") as f:
+        n_lines_raw = sum(1 for _ in f)
+    logger.info(f"Found {n_lines_raw} lines in raw data file.")
+
+    # Read data in chunks to avoid memory issues
+    train_ids, test_ids = [], []
+    lightweight_test = []  # We'll use this to create a gold submission later
+    with tqdm(total=n_lines_raw, desc="Splitting raw data -> train/test") as pbar:
+        for df in pd.read_json(raw / train_file, orient="records", lines=True, chunksize=1_000):
+            # Convert IDs to strings, Kaggle.com is inconsistent about this but strings make more sense
+            df["example_id"] = df["example_id"].astype(str)
+            new_train, new_test = train_test_split(df, test_size=0.1, random_state=0)
+
+            keys_to_keep = [
+                "example_id",
+                "question_text",
+                "document_text",
+                "long_answer_candidates",
+            ]
+            new_test_without_labels = new_test.copy()[keys_to_keep]
+
+            # Append lines to new train and test
+            with open(public_train_file, "a") as f:
+                f.write(new_train.to_json(orient="records", lines=True))
+            with open(private / "test.jsonl", "a") as f:
+                f.write(new_test.to_json(orient="records", lines=True))
+            with open(public / "simplified-nq-test.jsonl", "a") as f:
+                f.write(new_test_without_labels.to_json(orient="records", lines=True))
+
+            train_ids.extend(new_train["example_id"].tolist())
+            test_ids.extend(new_test["example_id"].tolist())
+            lightweight_test.append(
+                new_test.copy()[["example_id", "question_text", "annotations"]]
+            )  # For gold submission
+            pbar.update(len(df))
+
+    lightweight_test_df = pd.concat(lightweight_test, ignore_index=True)
+
+    assert len(train_ids) + len(test_ids) == n_lines_raw
+    assert len(lightweight_test_df) == len(test_ids)
+
+    # Create submission files for the original test set
+    create_submission_files(lightweight_test_df, public, private, len(test_ids))
+    logger.info("Finished creating original `public` and `private` directories.")
+
+    # --- STAGE 2: Create Validation Split from the new Train Set ---
+    # This section reads the train set created in Stage 1 and splits it again.
+    # The outputs are written to `public_val` and `private_val` directories.
+
+    logger.info("Counting lines in the created train file for validation split...")
+    with open(public_train_file, "r") as f:
+        n_lines_train = sum(1 for _ in f)
+    logger.info(f"Found {n_lines_train} lines in train file.")
+
+    train_val_ids, test_val_ids = [], []
+    lightweight_test_val = []  # For the validation set's gold submission
+    with tqdm(total=n_lines_train, desc="Splitting train data -> train_val/test_val") as pbar:
+        # Read the newly created train file in chunks for the second split
+        for df_train_chunk in pd.read_json(
+            public_train_file, orient="records", lines=True, chunksize=1_000
+        ):
+            # Replicate the splitting logic on the training data
+            new_train_val, new_test_val = train_test_split(
+                df_train_chunk, test_size=0.1, random_state=0
+            )
+
+            keys_to_keep = [
+                "example_id",
+                "question_text",
+                "document_text",
+                "long_answer_candidates",
+            ]
+            new_test_val_without_labels = new_test_val.copy()[keys_to_keep]
+
+            # Append lines to the new validation-split train and test files
+            with open(public_val / "simplified-nq-train.jsonl", "a") as f:
+                f.write(new_train_val.to_json(orient="records", lines=True))
+            with open(private_val / "test.jsonl", "a") as f:
+                f.write(new_test_val.to_json(orient="records", lines=True))
+            with open(public_val / "simplified-nq-test.jsonl", "a") as f:
+                f.write(new_test_val_without_labels.to_json(orient="records", lines=True))
+
+            train_val_ids.extend(new_train_val["example_id"].tolist())
+            test_val_ids.extend(new_test_val["example_id"].tolist())
+            lightweight_test_val.append(
+                new_test_val.copy()[["example_id", "question_text", "annotations"]]
+            )
+            pbar.update(len(df_train_chunk))
+
+    lightweight_test_val_df = pd.concat(lightweight_test_val, ignore_index=True)
+
+    assert len(train_val_ids) + len(test_val_ids) == n_lines_train
+    assert len(lightweight_test_val_df) == len(test_val_ids)
+
+    # Create submission files for the new validation set
+    create_submission_files(lightweight_test_val_df, public_val, private_val, len(test_val_ids))
+    logger.info("Finished creating new `public_val` and `private_val` directories.")

dslighting/registry/text-normalization-challenge-english-language/checksums.yaml

@@ -0,0 +1,5 @@
+private:
+  answers.csv: 9cdf7a9f064c7476c74b8e7b90c72398
+  sample_submission.csv: e4d1b5d86abed38e6ef37cb7996c3f2c
+public: {}
+zip: 49579a7cd2984841d1f09bea3824b22a

dslighting/registry/text-normalization-challenge-english-language/config.yaml

@@ -0,0 +1,22 @@
+id: text-normalization-challenge-english-language
+name: Text Normalization Challenge - English Language
+competition_type: simple
+awards_medals: true
+prizes:
+  - position: 1
+    value: 12000
+  - position: 2
+    value: 8000
+  - position: 3
+    value: 5000
+description: mlebench/competitions/text-normalization-challenge-english-language/description.md
+
+dataset:
+  answers: text-normalization-challenge-english-language/prepared/private/answers.csv
+  sample_submission: text-normalization-challenge-english-language/prepared/private/sample_submission.csv
+
+grader:
+  name: accuracy
+  grade_fn: mlebench.competitions.text-normalization-challenge-english-language.grade:grade
+
+preparer: mlebench.competitions.text-normalization-challenge-english-language.prepare:prepare

dslighting/registry/text-normalization-challenge-english-language/description.md

@@ -0,0 +1,77 @@
+# Overview
+
+## Description
+
+As many of us can attest, learning another language is tough. Picking up on nuances like slang, dates and times, and local expressions, can often be a distinguishing factor between proficiency and fluency. This challenge is even more difficult for a machine.
+
+Many speech and language applications, including text-to-speech synthesis (TTS) and automatic speech recognition (ASR), require text to be converted from written expressions into appropriate "spoken" forms. This is a process known as text normalization, and helps convert 12:47 to "twelve forty-seven" and $3.16 into "three dollars, sixteen cents."
+
+However, one of the biggest challenges when developing a TTS or ASR system for a new language is to develop and test the grammar for all these rules, a task that requires quite a bit of linguistic sophistication and native speaker intuition.
+
+```
+A    <self>
+baby    <self>
+giraffe    <self>
+is    <self>
+6ft    six feet
+tall    <self>
+and    <self>
+weighs    <self>
+150lb    one hundred fifty pounds
+.    sil
+```
+
+In this competition, you are challenged to automate the process of developing text normalization grammars via machine learning. This track will focus on English, while a separate will focus on Russian here: [Russian Text Normalization Challenge](https://www.kaggle.com/competitions/text-normalization-challenge-english-language/text-normalization-challenge-russian-language)
+
+### About the sponsor
+
+Google's Text Normalization Research Group conducts research and creates tools for the detection, normalization and denormalization of non-standard words such as abbreviations, numbers or currency expressions; and semiotic classes -- text tokens and token sequences that represent particular entities that are semantically constrained, such as measure phrases, addresses or dates. Applications of this work include text-to-speech synthesis, automatic speech recognition, and information extraction/retrieval.
+
+## Evaluation
+
+Submissions are evaluated on prediction accuracy (the total percent of correct tokens). The predicted and actual string must match exactly in order to count as correct. In other words, we are measuring sequence accuracy, in that any error in the output for a given token in the input sequence means that that error is wrong. For example, if the input is "145" and the predicted output is "one forty five" but the correct output is "one hundred forty five", this is counted as a single error.
+
+### Submission File
+
+For each token (id) in the test set, you must predict the normalized text. The file should contain a header and have the following format:
+
+```
+id,after
+0_0,"the"
+0_1,"quick"
+0_2,"fox"
+...
+```
+
+## Prizes
+
+Three cash prizes will be awarded to the top teams:
+
+- 1st place - $12,000
+- 2nd place - $8,000
+- 3rd place - $5,000
+
+## Timeline
+
+- **November 14, 2017** - Entry deadline. You must accept the competition rules before this date in order to compete.
+- **November 14, 2017** - Team Merger deadline. This is the last day participants may join or merge teams.
+- **November 14, 2017** - New test set released, leaderboard reset.
+- **November 20, 2017** - Final submission deadline.
+
+All deadlines are at 11:59 PM UTC on the corresponding day unless otherwise noted. The competition organizers reserve the right to update the contest timeline if they deem it necessary.
+
+## Citation
+
+Addison Howard, RichardSproat, wellformedness, Will Cukierski. (2017). Text Normalization Challenge - English Language. Kaggle. https://kaggle.com/competitions/text-normalization-challenge-english-language
+
+# Data
+
+## Dataset Description
+
+You are provided with a large corpus of text. Each sentence has a **sentence_id**. Each token within a sentence has a **token_id**. The **before** column contains the raw text, the **after** column contains the normalized text. The aim of the competition is to predict the **after** column for the test set. The training set contains an additional column, **class**, which is provided to show the token type. **This column is intentionally omitted from the test set**. In addition, there is an **id** column used in the submission format. This is formed by concatenating the **sentence_id** and **token_id** with an underscore (e.g. 123_5).
+
+### File descriptions
+
+- en_sample_submission.csv - a submission file showing the correct format
+- en_test.csv - the test set, does not contain the normalized text
+- en_train.csv - the training set, contains the normalized text

dslighting/registry/text-normalization-challenge-english-language/description_obfuscated.md

@@ -0,0 +1,27 @@
+# Task
+
+Create a model to normalize text (convert text from written expressions to a form suitable for text-to-speech systems, e.g. converting "$3.16" into "three dollars, sixteen cents").
+
+# Metric
+
+Prediction accuracy (the total percent of correct tokens). The predicted and actual string must match exactly in order to count as correct.
+
+# Submission Format
+
+For each token (id) in the test set, you must predict the normalized text. The file should contain a header and have the following format:
+
+```
+id,after
+0_0,"the"
+0_1,"quick"
+0_2,"fox"
+...
+```
+
+# Data
+
+You are provided with a large corpus of text. Each sentence has a **sentence_id**. Each token within a sentence has a **token_id**. The **before** column contains the raw text, the **after** column contains the normalized text. The training set contains an additional column, **class**, which is provided to show the token type. In addition, there is an **id** column used in the submission format. This is formed by concatenating the **sentence_id** and **token_id** with an underscore (e.g. 123_5).
+
+- en_sample_submission.csv - a submission file showing the correct format
+- en_test.csv - the test set, does not contain the normalized text
+- en_train.csv - the training set, contains the normalized text

dslighting/registry/text-normalization-challenge-english-language/grade.py

@@ -0,0 +1,49 @@
+import pandas as pd
+from sklearn.metrics import accuracy_score
+
+from mlebench.grade_helpers import InvalidSubmissionError
+
+
+def grade(submission: pd.DataFrame, answers: pd.DataFrame) -> float:
+    accuracy_inputs = prepare_for_metric(
+        submission=submission,
+        answers=answers,
+        target_column="after",
+        id_column="id",
+    )
+    return accuracy_score(y_pred=accuracy_inputs["y_pred"], y_true=accuracy_inputs["y_true"])
+
+
+def prepare_for_metric(
+    submission: pd.DataFrame,
+    answers: pd.DataFrame,
+    target_column: str,
+    id_column: str,
+) -> dict:
+
+    # Answers checks
+    assert target_column in answers.columns, f"Answers must have a `{target_column}` column"
+    assert id_column in answers.columns, f"Answers must have a `{id_column}` column"
+
+    # Submission checks
+    if len(submission) != len(answers):
+        raise InvalidSubmissionError("Submission must have the same length as the answers.")
+    if target_column not in submission.columns:
+        raise InvalidSubmissionError(f"Submission must have a `{target_column}` column")
+    if id_column not in submission.columns:
+        raise InvalidSubmissionError(f"Submission must have a `{id_column}` column")
+
+    # sort on id to ensure correct order
+    submission = submission.sort_values(id_column)
+    answers = answers.sort_values(id_column)
+
+    if (submission[id_column].values != answers[id_column].values).any():
+        raise InvalidSubmissionError(f"Submission and Answers `{id_column}`'s do not match")
+
+    submission[target_column] = submission[target_column].astype(str)
+    answers[target_column] = answers[target_column].astype(str)
+
+    y_pred = submission[target_column].to_numpy()
+    y_true = answers[target_column].to_numpy()
+
+    return {"y_true": y_true, "y_pred": y_pred}

dslighting/registry/text-normalization-challenge-english-language/prepare.py

@@ -0,0 +1,115 @@
+import csv
+import zipfile
+from pathlib import Path
+
+from sklearn.model_selection import train_test_split
+
+from mlebench.utils import compress, extract, read_csv
+
+
+def prepare(raw: Path, public: Path, private: Path):
+
+    # Extract
+    extract(raw / "en_test_2.csv.zip", raw)  # We only use the 2nd stage test set
+    extract(raw / "en_train.csv.zip", raw)
+    extract(raw / "en_sample_submission_2.csv.zip", raw)
+
+    # Create train and test splits from train set
+    old_train = read_csv(raw / "en_train.csv")
+
+    # We split so that we don't share any sentence_ids between train and test
+    # This gives us len(new_train) = 8924976 and len(answers) = 993465
+    # Original was len(old_train) = 9918441 and len(old_test) = 956046
+    unique_sentence_ids = old_train["sentence_id"].unique()
+    train_sentence_ids, test_sentence_ids = train_test_split(
+        unique_sentence_ids, test_size=0.1, random_state=0
+    )
+    new_train = old_train[old_train["sentence_id"].isin(train_sentence_ids)]
+    answers = old_train[old_train["sentence_id"].isin(test_sentence_ids)]
+    assert set(new_train["sentence_id"]).isdisjoint(
+        set(answers["sentence_id"])
+    ), f"sentence_id is not disjoint between train and test sets"
+
+    # "sentence_id" counts need to be reset for new_train and answers
+    new_train_id_mapping = {
+        old_id: new_id for new_id, old_id in enumerate(new_train["sentence_id"].unique())
+    }
+    new_train.loc[:, "sentence_id"] = new_train["sentence_id"].map(new_train_id_mapping)
+    answers_id_mapping = {
+        old_id: new_id for new_id, old_id in enumerate(answers["sentence_id"].unique())
+    }
+    answers.loc[:, "sentence_id"] = answers["sentence_id"].map(answers_id_mapping)
+
+    # Create new test set
+    new_test = answers.drop(["after", "class"], axis=1).copy()
+
+    # Reformat answers to match sample submission format
+    answers = answers[["sentence_id", "token_id", "after"]].copy()
+    answers["id"] = answers["sentence_id"].astype(str) + "_" + answers["token_id"].astype(str)
+    answers = answers[["id", "after"]]
+
+    # Create sample submission
+    sample_submission = new_test[["sentence_id", "token_id", "before"]].copy()
+    sample_submission["id"] = (
+        sample_submission["sentence_id"].astype(str)
+        + "_"
+        + sample_submission["token_id"].astype(str)
+    )
+    sample_submission["after"] = sample_submission["before"]
+    sample_submission = sample_submission[["id", "after"]]
+
+    # Checks
+    assert new_train.columns.tolist() == [
+        "sentence_id",
+        "token_id",
+        "class",
+        "before",
+        "after",
+    ], f"new_train.columns.tolist() == {new_train.columns.tolist()}"
+    assert new_test.columns.tolist() == [
+        "sentence_id",
+        "token_id",
+        "before",
+    ], f"new_test.columns.tolist() == {new_test.columns.tolist()}"
+    assert sample_submission.columns.tolist() == [
+        "id",
+        "after",
+    ], f"sample_submission.columns.tolist() == {sample_submission.columns.tolist()}"
+    assert answers.columns.tolist() == [
+        "id",
+        "after",
+    ], f"answers.columns.tolist() == {answers.columns.tolist()}"
+    assert len(new_test) + len(new_train) == len(
+        old_train
+    ), f"New train and test sets do not sum to old train set, got {len(new_test) + len(new_train)} and {len(old_train)}"
+
+    # Write CSVs
+    answers.to_csv(
+        private / "answers.csv", index=False, quotechar='"', quoting=csv.QUOTE_NONNUMERIC
+    )
+    sample_submission.to_csv(
+        private / "sample_submission.csv", index=False, quotechar='"', quoting=csv.QUOTE_NONNUMERIC
+    )
+    new_train.to_csv(
+        public / "en_train.csv", index=False, quotechar='"', quoting=csv.QUOTE_NONNUMERIC
+    )
+    new_test.to_csv(
+        public / "en_test_2.csv", index=False, quotechar='"', quoting=csv.QUOTE_NONNUMERIC
+    )
+    sample_submission.to_csv(
+        public / "en_sample_submission_2.csv",
+        index=False,
+        quotechar='"',
+        quoting=csv.QUOTE_NONNUMERIC,
+    )
+
+    # Zip up
+    with zipfile.ZipFile(public / "en_train.csv.zip", "w") as zipf:
+        zipf.write(public / "en_train.csv", arcname="en_train.csv")
+    with zipfile.ZipFile(public / "en_test_2.csv.zip", "w") as zipf:
+        zipf.write(public / "en_test_2.csv", arcname="en_test_2.csv")
+    with zipfile.ZipFile(public / "en_sample_submission_2.csv.zip", "w") as zipf:
+        zipf.write(public / "en_sample_submission_2.csv", arcname="en_sample_submission_2.csv")
+    (public / "en_train.csv").unlink()
+    (public / "en_test_2.csv").unlink()
+    (public / "en_sample_submission_2.csv").unlink()