dslighting 1.7.1__py3-none-any.whl → 1.7.8__py3-none-any.whl

mlebench/competitions/tensorflow2-question-answering/prepare_val.py

@@ -0,0 +1,187 @@
+from pathlib import Path
+
+import pandas as pd
+from sklearn.model_selection import train_test_split
+from tqdm import tqdm
+
+from mlebench.utils import get_logger
+
+logger = get_logger(__name__)
+
+
+def extract_string(document_text: str, start_token_idx: int, end_token_idx: int) -> str:
+    document_tokens = document_text.split(" ")
+    extract_tokens = document_tokens[start_token_idx:end_token_idx]
+    return " ".join(extract_tokens)
+
+
+def create_submission_files(
+    lightweight_test_df: pd.DataFrame, public_path: Path, private_path: Path, n_test_samples: int
+):
+    """
+    Creates the gold submission (private) and sample submission (public) files.
+    """
+    # Create a gold submission with columns "example_id", "PredictionString"
+    gold_rows = []
+    for idx, sample in tqdm(
+        lightweight_test_df.iterrows(),
+        total=len(lightweight_test_df),
+        desc=f"Creating submission files for {public_path.name}",
+    ):
+        sample = sample.to_dict()
+        assert len(sample["annotations"]) == 1
+        annotation = sample["annotations"][0]
+
+        # Create short answer
+        # Multiple answers are possible: yes_no_answer or one of short_answers
+        # We just take the first one
+        if annotation["yes_no_answer"] != "NONE":
+            answer = annotation["yes_no_answer"]
+        elif len(annotation["short_answers"]) > 0:
+            start_token = annotation["short_answers"][0]["start_token"]
+            end_token = annotation["short_answers"][0]["end_token"]
+            answer = f"{start_token}:{end_token}"
+        else:
+            answer = ""
+
+        gold_rows.append(
+            {"example_id": f"{sample['example_id']}_short", "PredictionString": answer}
+        )
+
+        # Create long answer
+        if annotation["long_answer"]["start_token"] != -1:
+            start_token = annotation["long_answer"]["start_token"]
+            end_token = annotation["long_answer"]["end_token"]
+            answer = f"{start_token}:{end_token}"
+        else:
+            answer = ""
+
+        gold_rows.append({"example_id": f"{sample['example_id']}_long", "PredictionString": answer})
+
+    gold_submission = pd.DataFrame(gold_rows)
+    gold_submission.to_csv(private_path / "gold_submission.csv", index=False)
+
+    # Sample submission
+    sample_submission = gold_submission.copy()
+    sample_submission["PredictionString"] = ""
+    sample_submission.to_csv(public_path / "sample_submission.csv", index=False)
+
+    assert len(gold_submission) == 2 * n_test_samples
+    assert len(sample_submission) == 2 * n_test_samples
+
+
+def prepare(raw: Path, public: Path, private: Path):
+    """
+    Splits the data in raw into public and private datasets with appropriate test/train splits.
+    Then, it further splits the created train set into a smaller train set and a validation set.
+    """
+    # Define and create paths for the new validation split
+    public_val = public.parent / "public_val"
+    private_val = private.parent / "private_val"
+    public_val.mkdir(exist_ok=True)
+    private_val.mkdir(exist_ok=True)
+
+    # --- STAGE 1: Create Original Train/Test Split ---
+    # This section creates the original `public` and `private` outputs.
+
+    train_file = "simplified-nq-train.jsonl"
+    public_train_file = public / "simplified-nq-train.jsonl"
+
+    logger.info("Counting lines in raw data file...")
+    with open(raw / train_file, "r") as f:
+        n_lines_raw = sum(1 for _ in f)
+    logger.info(f"Found {n_lines_raw} lines in raw data file.")
+
+    # Read data in chunks to avoid memory issues
+    train_ids, test_ids = [], []
+    lightweight_test = []  # We'll use this to create a gold submission later
+    with tqdm(total=n_lines_raw, desc="Splitting raw data -> train/test") as pbar:
+        for df in pd.read_json(raw / train_file, orient="records", lines=True, chunksize=1_000):
+            # Convert IDs to strings, Kaggle.com is inconsistent about this but strings make more sense
+            df["example_id"] = df["example_id"].astype(str)
+            new_train, new_test = train_test_split(df, test_size=0.1, random_state=0)
+
+            keys_to_keep = [
+                "example_id",
+                "question_text",
+                "document_text",
+                "long_answer_candidates",
+            ]
+            new_test_without_labels = new_test.copy()[keys_to_keep]
+
+            # Append lines to new train and test
+            with open(public_train_file, "a") as f:
+                f.write(new_train.to_json(orient="records", lines=True))
+            with open(private / "test.jsonl", "a") as f:
+                f.write(new_test.to_json(orient="records", lines=True))
+            with open(public / "simplified-nq-test.jsonl", "a") as f:
+                f.write(new_test_without_labels.to_json(orient="records", lines=True))
+
+            train_ids.extend(new_train["example_id"].tolist())
+            test_ids.extend(new_test["example_id"].tolist())
+            lightweight_test.append(
+                new_test.copy()[["example_id", "question_text", "annotations"]]
+            )  # For gold submission
+            pbar.update(len(df))
+
+    lightweight_test_df = pd.concat(lightweight_test, ignore_index=True)
+
+    assert len(train_ids) + len(test_ids) == n_lines_raw
+    assert len(lightweight_test_df) == len(test_ids)
+
+    # Create submission files for the original test set
+    create_submission_files(lightweight_test_df, public, private, len(test_ids))
+    logger.info("Finished creating original `public` and `private` directories.")
+
+    # --- STAGE 2: Create Validation Split from the new Train Set ---
+    # This section reads the train set created in Stage 1 and splits it again.
+    # The outputs are written to `public_val` and `private_val` directories.
+
+    logger.info("Counting lines in the created train file for validation split...")
+    with open(public_train_file, "r") as f:
+        n_lines_train = sum(1 for _ in f)
+    logger.info(f"Found {n_lines_train} lines in train file.")
+
+    train_val_ids, test_val_ids = [], []
+    lightweight_test_val = []  # For the validation set's gold submission
+    with tqdm(total=n_lines_train, desc="Splitting train data -> train_val/test_val") as pbar:
+        # Read the newly created train file in chunks for the second split
+        for df_train_chunk in pd.read_json(
+            public_train_file, orient="records", lines=True, chunksize=1_000
+        ):
+            # Replicate the splitting logic on the training data
+            new_train_val, new_test_val = train_test_split(
+                df_train_chunk, test_size=0.1, random_state=0
+            )
+
+            keys_to_keep = [
+                "example_id",
+                "question_text",
+                "document_text",
+                "long_answer_candidates",
+            ]
+            new_test_val_without_labels = new_test_val.copy()[keys_to_keep]
+
+            # Append lines to the new validation-split train and test files
+            with open(public_val / "simplified-nq-train.jsonl", "a") as f:
+                f.write(new_train_val.to_json(orient="records", lines=True))
+            with open(private_val / "test.jsonl", "a") as f:
+                f.write(new_test_val.to_json(orient="records", lines=True))
+            with open(public_val / "simplified-nq-test.jsonl", "a") as f:
+                f.write(new_test_val_without_labels.to_json(orient="records", lines=True))
+
+            train_val_ids.extend(new_train_val["example_id"].tolist())
+            test_val_ids.extend(new_test_val["example_id"].tolist())
+            lightweight_test_val.append(
+                new_test_val.copy()[["example_id", "question_text", "annotations"]]
+            )
+            pbar.update(len(df_train_chunk))
+
+    lightweight_test_val_df = pd.concat(lightweight_test_val, ignore_index=True)
+
+    assert len(train_val_ids) + len(test_val_ids) == n_lines_train
+    assert len(lightweight_test_val_df) == len(test_val_ids)
+
+    # Create submission files for the new validation set
+    create_submission_files(lightweight_test_val_df, public_val, private_val, len(test_val_ids))
+    logger.info("Finished creating new `public_val` and `private_val` directories.")

mlebench/competitions/text-normalization-challenge-english-language/grade.py

@@ -0,0 +1,49 @@
+import pandas as pd
+from sklearn.metrics import accuracy_score
+
+from mlebench.grade_helpers import InvalidSubmissionError
+
+
+def grade(submission: pd.DataFrame, answers: pd.DataFrame) -> float:
+    accuracy_inputs = prepare_for_metric(
+        submission=submission,
+        answers=answers,
+        target_column="after",
+        id_column="id",
+    )
+    return accuracy_score(y_pred=accuracy_inputs["y_pred"], y_true=accuracy_inputs["y_true"])
+
+
+def prepare_for_metric(
+    submission: pd.DataFrame,
+    answers: pd.DataFrame,
+    target_column: str,
+    id_column: str,
+) -> dict:
+
+    # Answers checks
+    assert target_column in answers.columns, f"Answers must have a `{target_column}` column"
+    assert id_column in answers.columns, f"Answers must have a `{id_column}` column"
+
+    # Submission checks
+    if len(submission) != len(answers):
+        raise InvalidSubmissionError("Submission must have the same length as the answers.")
+    if target_column not in submission.columns:
+        raise InvalidSubmissionError(f"Submission must have a `{target_column}` column")
+    if id_column not in submission.columns:
+        raise InvalidSubmissionError(f"Submission must have a `{id_column}` column")
+
+    # sort on id to ensure correct order
+    submission = submission.sort_values(id_column)
+    answers = answers.sort_values(id_column)
+
+    if (submission[id_column].values != answers[id_column].values).any():
+        raise InvalidSubmissionError(f"Submission and Answers `{id_column}`'s do not match")
+
+    submission[target_column] = submission[target_column].astype(str)
+    answers[target_column] = answers[target_column].astype(str)
+
+    y_pred = submission[target_column].to_numpy()
+    y_true = answers[target_column].to_numpy()
+
+    return {"y_true": y_true, "y_pred": y_pred}

mlebench/competitions/text-normalization-challenge-english-language/prepare.py

@@ -0,0 +1,115 @@
+import csv
+import zipfile
+from pathlib import Path
+
+from sklearn.model_selection import train_test_split
+
+from mlebench.utils import compress, extract, read_csv
+
+
+def prepare(raw: Path, public: Path, private: Path):
+
+    # Extract
+    extract(raw / "en_test_2.csv.zip", raw)  # We only use the 2nd stage test set
+    extract(raw / "en_train.csv.zip", raw)
+    extract(raw / "en_sample_submission_2.csv.zip", raw)
+
+    # Create train and test splits from train set
+    old_train = read_csv(raw / "en_train.csv")
+
+    # We split so that we don't share any sentence_ids between train and test
+    # This gives us len(new_train) = 8924976 and len(answers) = 993465
+    # Original was len(old_train) = 9918441 and len(old_test) = 956046
+    unique_sentence_ids = old_train["sentence_id"].unique()
+    train_sentence_ids, test_sentence_ids = train_test_split(
+        unique_sentence_ids, test_size=0.1, random_state=0
+    )
+    new_train = old_train[old_train["sentence_id"].isin(train_sentence_ids)]
+    answers = old_train[old_train["sentence_id"].isin(test_sentence_ids)]
+    assert set(new_train["sentence_id"]).isdisjoint(
+        set(answers["sentence_id"])
+    ), f"sentence_id is not disjoint between train and test sets"
+
+    # "sentence_id" counts need to be reset for new_train and answers
+    new_train_id_mapping = {
+        old_id: new_id for new_id, old_id in enumerate(new_train["sentence_id"].unique())
+    }
+    new_train.loc[:, "sentence_id"] = new_train["sentence_id"].map(new_train_id_mapping)
+    answers_id_mapping = {
+        old_id: new_id for new_id, old_id in enumerate(answers["sentence_id"].unique())
+    }
+    answers.loc[:, "sentence_id"] = answers["sentence_id"].map(answers_id_mapping)
+
+    # Create new test set
+    new_test = answers.drop(["after", "class"], axis=1).copy()
+
+    # Reformat answers to match sample submission format
+    answers = answers[["sentence_id", "token_id", "after"]].copy()
+    answers["id"] = answers["sentence_id"].astype(str) + "_" + answers["token_id"].astype(str)
+    answers = answers[["id", "after"]]
+
+    # Create sample submission
+    sample_submission = new_test[["sentence_id", "token_id", "before"]].copy()
+    sample_submission["id"] = (
+        sample_submission["sentence_id"].astype(str)
+        + "_"
+        + sample_submission["token_id"].astype(str)
+    )
+    sample_submission["after"] = sample_submission["before"]
+    sample_submission = sample_submission[["id", "after"]]
+
+    # Checks
+    assert new_train.columns.tolist() == [
+        "sentence_id",
+        "token_id",
+        "class",
+        "before",
+        "after",
+    ], f"new_train.columns.tolist() == {new_train.columns.tolist()}"
+    assert new_test.columns.tolist() == [
+        "sentence_id",
+        "token_id",
+        "before",
+    ], f"new_test.columns.tolist() == {new_test.columns.tolist()}"
+    assert sample_submission.columns.tolist() == [
+        "id",
+        "after",
+    ], f"sample_submission.columns.tolist() == {sample_submission.columns.tolist()}"
+    assert answers.columns.tolist() == [
+        "id",
+        "after",
+    ], f"answers.columns.tolist() == {answers.columns.tolist()}"
+    assert len(new_test) + len(new_train) == len(
+        old_train
+    ), f"New train and test sets do not sum to old train set, got {len(new_test) + len(new_train)} and {len(old_train)}"
+
+    # Write CSVs
+    answers.to_csv(
+        private / "answers.csv", index=False, quotechar='"', quoting=csv.QUOTE_NONNUMERIC
+    )
+    sample_submission.to_csv(
+        private / "sample_submission.csv", index=False, quotechar='"', quoting=csv.QUOTE_NONNUMERIC
+    )
+    new_train.to_csv(
+        public / "en_train.csv", index=False, quotechar='"', quoting=csv.QUOTE_NONNUMERIC
+    )
+    new_test.to_csv(
+        public / "en_test_2.csv", index=False, quotechar='"', quoting=csv.QUOTE_NONNUMERIC
+    )
+    sample_submission.to_csv(
+        public / "en_sample_submission_2.csv",
+        index=False,
+        quotechar='"',
+        quoting=csv.QUOTE_NONNUMERIC,
+    )
+
+    # Zip up
+    with zipfile.ZipFile(public / "en_train.csv.zip", "w") as zipf:
+        zipf.write(public / "en_train.csv", arcname="en_train.csv")
+    with zipfile.ZipFile(public / "en_test_2.csv.zip", "w") as zipf:
+        zipf.write(public / "en_test_2.csv", arcname="en_test_2.csv")
+    with zipfile.ZipFile(public / "en_sample_submission_2.csv.zip", "w") as zipf:
+        zipf.write(public / "en_sample_submission_2.csv", arcname="en_sample_submission_2.csv")
+    (public / "en_train.csv").unlink()
+    (public / "en_test_2.csv").unlink()
+    (public / "en_sample_submission_2.csv").unlink()

mlebench/competitions/text-normalization-challenge-english-language/prepare_val.py

@@ -0,0 +1,213 @@
+import csv
+import zipfile
+from pathlib import Path
+from typing import Tuple
+
+import pandas as pd
+from sklearn.model_selection import train_test_split
+
+from mlebench.utils import extract, read_csv
+
+
+def _split_and_process(
+    input_df: pd.DataFrame, test_size: float, random_state: int
+) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame, pd.DataFrame]:
+    """
+    Splits a DataFrame into train/test sets and processes them for the competition format.
+
+    This function encapsulates the core logic of splitting data based on unique sentence IDs,
+    re-indexing the new splits, and creating the corresponding test data, answers, and
+    sample submission files.
+
+    Args:
+        input_df: The DataFrame to split.
+        test_size: The proportion of the dataset to allocate to the test split.
+        random_state: The seed used by the random number generator.
+
+    Returns:
+        A tuple containing:
+        - new_train_df: The processed training data.
+        - new_test_df: The processed, unlabeled test data.
+        - answers_df: The ground truth labels for the test data.
+        - submission_df: A sample submission file for the test data.
+    """
+    # We split so that we don't share any sentence_ids between train and test
+    unique_sentence_ids = input_df["sentence_id"].unique()
+    train_sentence_ids, test_sentence_ids = train_test_split(
+        unique_sentence_ids, test_size=test_size, random_state=random_state
+    )
+    new_train_df = input_df[input_df["sentence_id"].isin(train_sentence_ids)].copy()
+    answers_df = input_df[input_df["sentence_id"].isin(test_sentence_ids)].copy()
+    assert set(new_train_df["sentence_id"]).isdisjoint(
+        set(answers_df["sentence_id"])
+    ), f"sentence_id is not disjoint between train and test sets"
+
+    # "sentence_id" counts need to be reset for new_train_df and answers_df
+    new_train_id_mapping = {
+        old_id: new_id
+        for new_id, old_id in enumerate(new_train_df["sentence_id"].unique())
+    }
+    new_train_df.loc[:, "sentence_id"] = new_train_df["sentence_id"].map(
+        new_train_id_mapping
+    )
+    answers_id_mapping = {
+        old_id: new_id
+        for new_id, old_id in enumerate(answers_df["sentence_id"].unique())
+    }
+    answers_df.loc[:, "sentence_id"] = answers_df["sentence_id"].map(answers_id_mapping)
+
+    # Create new test set
+    new_test_df = answers_df.drop(["after", "class"], axis=1).copy()
+
+    # Reformat answers to match sample submission format
+    answers_formatted = answers_df[["sentence_id", "token_id", "after"]].copy()
+    answers_formatted["id"] = (
+        answers_formatted["sentence_id"].astype(str)
+        + "_"
+        + answers_formatted["token_id"].astype(str)
+    )
+    answers_formatted = answers_formatted[["id", "after"]]
+
+    # Create sample submission
+    submission_df = new_test_df[["sentence_id", "token_id", "before"]].copy()
+    submission_df["id"] = (
+        submission_df["sentence_id"].astype(str)
+        + "_"
+        + submission_df["token_id"].astype(str)
+    )
+    submission_df["after"] = submission_df["before"]
+    submission_df = submission_df[["id", "after"]]
+
+    # Checks
+    assert new_train_df.columns.tolist() == [
+        "sentence_id",
+        "token_id",
+        "class",
+        "before",
+        "after",
+    ], f"new_train_df.columns.tolist() == {new_train_df.columns.tolist()}"
+    assert new_test_df.columns.tolist() == [
+        "sentence_id",
+        "token_id",
+        "before",
+    ], f"new_test_df.columns.tolist() == {new_test_df.columns.tolist()}"
+    assert submission_df.columns.tolist() == [
+        "id",
+        "after",
+    ], f"submission_df.columns.tolist() == {submission_df.columns.tolist()}"
+    assert answers_formatted.columns.tolist() == [
+        "id",
+        "after",
+    ], f"answers_formatted.columns.tolist() == {answers_formatted.columns.tolist()}"
+    assert len(new_test_df) + len(new_train_df) == len(
+        input_df
+    ), f"New train and test sets do not sum to old train set, got {len(new_test_df) + len(new_train_df)} and {len(input_df)}"
+
+    return new_train_df, new_test_df, answers_formatted, submission_df
+
+
+def _write_and_zip_outputs(
+    public_dir: Path,
+    private_dir: Path,
+    train_df: pd.DataFrame,
+    test_df: pd.DataFrame,
+    answers_df: pd.DataFrame,
+    submission_df: pd.DataFrame,
+):
+    """
+    Writes the processed DataFrames to the specified directories and creates zip archives.
+    """
+    public_dir.mkdir(exist_ok=True, parents=True)
+    private_dir.mkdir(exist_ok=True, parents=True)
+
+    # Define file paths
+    answers_path = private_dir / "answers.csv"
+    private_submission_path = private_dir / "sample_submission.csv"
+    public_train_path = public_dir / "en_train.csv"
+    public_test_path = public_dir / "en_test_2.csv"
+    public_submission_path = public_dir / "en_sample_submission_2.csv"
+
+    # Write CSVs
+    answers_df.to_csv(
+        answers_path, index=False, quotechar='"', quoting=csv.QUOTE_NONNUMERIC
+    )
+    submission_df.to_csv(
+        private_submission_path,
+        index=False,
+        quotechar='"',
+        quoting=csv.QUOTE_NONNUMERIC,
+    )
+    train_df.to_csv(
+        public_train_path, index=False, quotechar='"', quoting=csv.QUOTE_NONNUMERIC
+    )
+    test_df.to_csv(
+        public_test_path, index=False, quotechar='"', quoting=csv.QUOTE_NONNUMERIC
+    )
+    submission_df.to_csv(
+        public_submission_path,
+        index=False,
+        quotechar='"',
+        quoting=csv.QUOTE_NONNUMERIC,
+    )
+
+    # Zip up public files
+    with zipfile.ZipFile(public_dir / "en_train.csv.zip", "w", zipfile.ZIP_DEFLATED) as zipf:
+        zipf.write(public_train_path, arcname="en_train.csv")
+    with zipfile.ZipFile(public_dir / "en_test_2.csv.zip", "w", zipfile.ZIP_DEFLATED) as zipf:
+        zipf.write(public_test_path, arcname="en_test_2.csv")
+    with zipfile.ZipFile(
+        public_dir / "en_sample_submission_2.csv.zip", "w", zipfile.ZIP_DEFLATED
+    ) as zipf:
+        zipf.write(public_submission_path, arcname="en_sample_submission_2.csv")
+
+    # Clean up intermediate CSVs
+    public_train_path.unlink()
+    public_test_path.unlink()
+    public_submission_path.unlink()
+
+
+def prepare(raw: Path, public: Path, private: Path):
+
+    # Extract
+    extract(raw / "en_test_2.csv.zip", raw)  # We only use the 2nd stage test set
+    extract(raw / "en_train.csv.zip", raw)
+    extract(raw / "en_sample_submission_2.csv.zip", raw)
+
+    # Load original training data
+    old_train = read_csv(raw / "en_train.csv")
+
+    # --- Stage 1: Create the original train/test split ---
+    new_train, new_test, answers, sample_submission = _split_and_process(
+        input_df=old_train, test_size=0.1, random_state=0
+    )
+
+    # Write the original public and private directory files
+    _write_and_zip_outputs(
+        public_dir=public,
+        private_dir=private,
+        train_df=new_train,
+        test_df=new_test,
+        answers_df=answers,
+        submission_df=sample_submission,
+    )
+
+    # --- Stage 2: Create the new train_val/test_val split ---
+    # Define new output directories for the validation split
+    public_val = public.parent / "public_val"
+    private_val = private.parent / "private_val"
+
+    # Create the validation split from the 'new_train' data generated in Stage 1
+    # We use the same splitting logic and parameters to ensure consistency.
+    train_val, test_val, answers_val, submission_val = _split_and_process(
+        input_df=new_train, test_size=0.1, random_state=0
+    )
+
+    # Write the validation public and private directory files
+    _write_and_zip_outputs(
+        public_dir=public_val,
+        private_dir=private_val,
+        train_df=train_val,
+        test_df=test_val,
+        answers_df=answers_val,
+        submission_df=submission_val,
+    )

mlebench/competitions/text-normalization-challenge-russian-language/grade.py

@@ -0,0 +1,49 @@
+import pandas as pd
+from sklearn.metrics import accuracy_score
+
+from mlebench.grade_helpers import InvalidSubmissionError
+
+
+def grade(submission: pd.DataFrame, answers: pd.DataFrame) -> float:
+    accuracy_inputs = prepare_for_metric(
+        submission=submission,
+        answers=answers,
+        target_column="after",
+        id_column="id",
+    )
+    return accuracy_score(y_pred=accuracy_inputs["y_pred"], y_true=accuracy_inputs["y_true"])
+
+
+def prepare_for_metric(
+    submission: pd.DataFrame,
+    answers: pd.DataFrame,
+    target_column: str,
+    id_column: str,
+) -> dict:
+
+    # Answers checks
+    assert target_column in answers.columns, f"Answers must have a `{target_column}` column"
+    assert id_column in answers.columns, f"Answers must have a `{id_column}` column"
+
+    # Submission checks
+    if len(submission) != len(answers):
+        raise InvalidSubmissionError("Submission must have the same length as the answers.")
+    if target_column not in submission.columns:
+        raise InvalidSubmissionError(f"Submission must have a `{target_column}` column")
+    if id_column not in submission.columns:
+        raise InvalidSubmissionError(f"Submission must have a `{id_column}` column")
+
+    # sort on id to ensure correct order
+    submission = submission.sort_values(id_column)
+    answers = answers.sort_values(id_column)
+
+    if (submission[id_column].values != answers[id_column].values).any():
+        raise InvalidSubmissionError(f"Submission and Answers `{id_column}`'s do not match")
+
+    submission[target_column] = submission[target_column].astype(str)
+    answers[target_column] = answers[target_column].astype(str)
+
+    y_pred = submission[target_column].to_numpy()
+    y_true = answers[target_column].to_numpy()
+
+    return {"y_true": y_true, "y_pred": y_pred}