dslighting 1.3.9__py3-none-any.whl → 1.7.1__py3-none-any.whl

dslighting/registry/aerial-cactus-identification/prepare_val.py

@@ -0,0 +1,133 @@
+import shutil
+from pathlib import Path
+import pandas as pd
+
+from sklearn.model_selection import train_test_split
+from tqdm.auto import tqdm
+
+from mlebench.utils import compress, extract, read_csv
+
+
+def _process_and_save_split(
+    train_df: pd.DataFrame,
+    test_df: pd.DataFrame,
+    public_dir: Path,
+    private_dir: Path,
+    raw_image_dir: Path,
+    original_df: pd.DataFrame,
+):
+    """
+    Helper function to process a single train/test split.
+    It saves all required CSVs, copies and compresses images, and runs assertions.
+    """
+    public_dir.mkdir(parents=True, exist_ok=True)
+    private_dir.mkdir(parents=True, exist_ok=True)
+    
+    # Save the dataframes
+    train_df.to_csv(public_dir / "train.csv", index=False)
+    test_df.to_csv(private_dir / "test.csv", index=False)
+
+    sample_submission = test_df.copy()
+    sample_submission["has_cactus"] = 0.5
+    sample_submission.to_csv(public_dir / "sample_submission.csv", index=False)
+
+    # Prepare temporary directories for images
+    temp_train_img_dir = public_dir / "train"
+    temp_test_img_dir = public_dir / "test"
+    temp_train_img_dir.mkdir(parents=True, exist_ok=True)
+    temp_test_img_dir.mkdir(parents=True, exist_ok=True)
+
+    # Copy images for the current split
+    for image_id in tqdm(train_df["id"], desc=f"Copying train images to {public_dir.name}", total=len(train_df)):
+        shutil.copy(raw_image_dir / image_id, temp_train_img_dir / image_id)
+
+    for image_id in tqdm(test_df["id"], desc=f"Copying test images to {public_dir.name}", total=len(test_df)):
+        shutil.copy(raw_image_dir / image_id, temp_test_img_dir / image_id)
+
+    # and then recompress
+    compress(temp_train_img_dir, public_dir / "train.zip")
+    compress(temp_test_img_dir, public_dir / "test.zip")
+
+    # and cleanup temporary image directories
+    shutil.rmtree(temp_train_img_dir)
+    shutil.rmtree(temp_test_img_dir)
+
+    # checks for the current split
+    assert (public_dir / "train.zip").exists(), f"{public_dir.name}/train.zip should exist"
+    assert (public_dir / "test.zip").exists(), f"{public_dir.name}/test.zip should exist"
+
+    assert len(train_df) + len(test_df) == len(
+        original_df
+    ), "The lengths of the splits should add up to the original"
+    assert len(train_df) > len(test_df), "The train set should be larger than the test set"
+    assert len(test_df) == len(
+        sample_submission
+    ), "The test set should match the sample submission"
+
+    assert (
+        train_df.columns.tolist()
+        == test_df.columns.tolist()
+        == original_df.columns.tolist()
+        == sample_submission.columns.tolist()
+    ), "All dataframes should have the same columns, i.e. ['id', 'has_cactus']"
+
+    assert set(train_df["id"]).isdisjoint(test_df["id"]), "Train and test ids should not overlap"
+    assert set(test_df["id"]) == set(
+        sample_submission["id"]
+    ), "Test and sample_submission ids should match"
+
+    assert train_df["id"].nunique() == len(train_df), "There should be no duplicate ids in train"
+    assert test_df["id"].nunique() == len(test_df), "There should be no duplicate ids in test"
+
+
+def prepare(raw: Path, public: Path, private: Path):
+
+    old_train = read_csv(raw / "train.csv")
+    
+    # need to split the train.zip into train.zip and test.zip; to do so need to extract first
+    extract(raw / "train.zip", raw)
+    raw_image_dir = raw / "train"
+
+    # --- 1. Original Split: Create main train and test sets ---
+    # This block creates the original public/private outputs, which must remain unchanged.
+    
+    # 4000 / (4000 + 17500) -> test_size is ~0.19
+    new_train, new_test = train_test_split(old_train, test_size=0.19, random_state=0)
+    
+    _process_and_save_split(
+        train_df=new_train,
+        test_df=new_test,
+        public_dir=public,
+        private_dir=private,
+        raw_image_dir=raw_image_dir,
+        original_df=old_train,
+    )
+    
+    # --- 2. New Validation Split: Create a validation set from the main train set ---
+    # This block creates the new public_val/private_val outputs.
+    
+    # Define paths for the new validation split artifacts
+    public_val = public.parent / "public_val"
+    private_val = private.parent / "private_val"
+
+    # To make the new validation set (test_val) have the same size as the original
+    # test set (new_test), we calculate the required test_size proportion.
+    test_size_for_val_split = len(new_test) / len(new_train)
+    
+    # Split the main training set (new_train) into a smaller training set (train_val)
+    # and a validation set (test_val), using the same random_state for consistency.
+    train_val, test_val = train_test_split(new_train, test_size=test_size_for_val_split, random_state=0)
+
+    _process_and_save_split(
+        train_df=train_val,
+        test_df=test_val,
+        public_dir=public_val,
+        private_dir=private_val,
+        raw_image_dir=raw_image_dir,
+        original_df=new_train,
+    )
+
+    # --- Final Cleanup ---
+    # and cleanup the extracted raw image directory, which is no longer needed
+    shutil.rmtree(raw_image_dir)
+    assert not (raw / "train").exists(), "raw/train/ should not exist"

dslighting/registry/alaska2-image-steganalysis/checksums.yaml

@@ -0,0 +1,5 @@
+private:
+  test.csv: f90b06076f750fb56ae35c3df715ada1
+public:
+  sample_submission.csv: f4623502821bbd1399ecc8f5d66fc243
+zip: 774e19a747b3d48d742334b9021c9915

dslighting/registry/alaska2-image-steganalysis/config.yaml

@@ -0,0 +1,22 @@
+id: alaska2-image-steganalysis
+name: ALASKA2 Image Steganalysis
+competition_type: simple
+awards_medals: true
+prizes:
+  - position: 1
+    value: 12000
+  - position: 2
+    value: 8000
+  - position: 3
+    value: 5000
+description: mlebench/competitions/alaska2-image-steganalysis/description.md
+
+dataset:
+  answers: alaska2-image-steganalysis/prepared/private/test.csv
+  sample_submission: alaska2-image-steganalysis/prepared/public/sample_submission.csv
+
+grader:
+  name: weighted-auroc
+  grade_fn: mlebench.competitions.alaska2-image-steganalysis.grade:grade
+
+preparer: mlebench.competitions.alaska2-image-steganalysis.prepare:prepare

dslighting/registry/alaska2-image-steganalysis/description.md

@@ -0,0 +1,91 @@
+# Overview
+
+## Description
+
+That file you downloaded may contain hidden messages that aren't part of its regular contents. The same technology employed for digital watermarking is also misused by crime rings. Law enforcement must now use steganalysis to detect these messages as part of their investigations. Machine learning is an important tool in the discovery of this secret data.
+
+![](https://storage.googleapis.com/kaggle-media/competitions/UTT/Kaggle_ALASKA.png)
+
+Current methods produce unreliable results, raising false alarms. One reason for inaccuracy is the many different devices and processing combinations. Yet, detection models are trained on a homogeneous dataset. To increase accuracy, researchers must put data hidden within digital images "into the wild" (hence the name ALASKA) to mimic real world conditions.
+
+In the competition, you'll create an efficient and reliable method to detect secret data hidden within innocuous-seeming digital images. Rather than limiting the data source, these images have been acquired with as many as 50 different cameras (from smartphone to full-format high end) and processed in different fashions. Successful entries will include robust detection algorithms with minimal false positives.
+
+The IEEE WIFS (Workshop on Information Forensics and Security) is eager to make this happen again, as a follow up to the [ALASKA#1 Challenge](https://alaska.utt.fr/#HallOfFame). WIFS is an annual event where researchers gather to discuss emerging challenges, exchange fresh ideas, and share state-of-the-art results and technical expertise in the areas of information security and forensics. WIFS has teamed up with Troyes University of Technology, CRIStAL Lab, Lille University, and CNRS to enable more accurate steganalysis.
+
+Law enforcement officers need better methods to combat criminals using hidden messages. The data science community and other researchers can help with better automated detection. More accurate methods could help catch criminals whose communications are hidden in plain sight.
+
+The challenge is organized by Rémi COGRANNE (UTT), Patrick BAS (CRIStAL / CNRS) and Quentin Giboulot (UTT) ; in addition to Kaggle, we have been greatly helped by the following sponsors:
+
+![](https://www.googleapis.com/download/storage/v1/b/kaggle-user-content/o/inbox%2F1951250%2F3a24a302d6a43d42087769d57048b566%2Flogo_UTT_CRIStAL.png?generation=1588208625932740&alt=media) ![](https://www.googleapis.com/download/storage/v1/b/kaggle-user-content/o/inbox%2F1951250%2Fa5c6f8c179e51a4048b581191166425d%2FLogoCNRS_SPS.png?generation=1588208648755636&alt=media)
+
+## Evaluation
+
+In order to focus on reliable detection with an emphasis on low false-alarm rate, submissions are evaluated on the weighted AUC. To calculate the weighted AUC, each region of the ROC curve is weighted according to these chosen parameters:
+
+```
+tpr_thresholds = [0.0, 0.4, 1.0]
+weights = [2, 1]
+```
+
+In other words, the area between the true positive rate of 0 and 0.4 is weighted 2X, the area between 0.4 and 1 is now weighed (1X). The total area is normalized by the sum of weights such that the final weighted AUC is between 0 and 1.
+
+The figure below illustrates the different areas. This example also shows that the submission #2 has the lowest AUC but is ranked first since it outperforms its competitors for low False Positive Rate (FPR). On the opposite, the submission #3 has the highest AUC but would have been ranked third.
+
+![](https://www.googleapis.com/download/storage/v1/b/kaggle-user-content/o/inbox%2F1951250%2Ff250ff6a4e04bac332fa14d539ed813e%2FKaggle.png?generation=1588207999884987&alt=media)
+
+## Submission File
+
+For each `Id` (image) in the test set, you must provide a score that indicates how likely this image contains hidden data: the higher the score, the more it is assumed that image contains secret data. The file should contain a header and have the following format:
+
+```
+Id,Label
+0001.jpg,0.1
+0002.jpg,0.99
+0003.jpg,1.2
+0004.jpg,-2.2
+etc.
+```
+
+## Timeline
+
+- July 13, 2020 - Entry deadline. You must accept the competition rules before this date in order to compete.
+
+- July 13, 2020 - Team Merger deadline. This is the last day participants may join or merge teams.
+
+- July 20, 2020 - Final submission deadline.
+
+All deadlines are at 11:59 PM UTC on the corresponding day unless otherwise noted. The competition organizers reserve the right to update the contest timeline if they deem it necessary.
+
+## Prizes
+
+- 1st Place - $12,000
+- 2nd Place - $8,000
+- 3rd Place - $5,000
+
+## Citation
+
+Addison Howard, GIBOULOT Quentin, PatrickFrenchie, Rémi Cogranne, Will Cukierski. (2020). ALASKA2 Image Steganalysis. Kaggle. https://kaggle.com/competitions/alaska2-image-steganalysis
+
+
+# Data
+
+## Dataset Description
+
+This dataset contains a large number of unaltered images, called the "Cover" image, as well as corresponding examples in which information has been hidden using one of three steganography algorithms (JMiPOD, JUNIWARD, UERD).\
+The goal of the competition is to determine which of the images in the test set (`Test/`) have hidden messages embedded.
+
+Note that in order to make the competition more realistic the length of hidden messages (the payload) will not be provided. The only available information on the test set is:
+
+1. Each embedding algorithm is used with the same probability.
+2. The payload (message length) is adjusted such that the "difficulty" is approximately the same regardless the content of the image. Images with smooth content are used to hide shorter messages while highly textured images will be used to hide more secret bits. The payload is adjusted in the same manner for testing and training sets.
+3. The average message length is 0.4 bit per non-zero AC DCT coefficient.
+4. The images are all compressed with one of the three following JPEG quality factors: 95, 90 or 75.
+
+## Files
+
+- `Cover/` contains 75k unaltered images meant for use in training.
+- `JMiPOD/` contains 75k examples of the JMiPOD algorithm applied to the cover images.
+- `JUNIWARD/`contains 75k examples of the JUNIWARD algorithm applied to the cover images.
+- `UERD/` contains 75k examples of the UERD algorithm applied to the cover images.
+- `Test/` contains 5k test set images. These are the images for which you are predicting.
+- `sample_submission.csv` contains an example submission in the correct format.

dslighting/registry/alaska2-image-steganalysis/description_obfuscated.md

@@ -0,0 +1,44 @@
+# Task
+
+Determine which of the images have hidden messages embedded using one of three steganography algorithms (JMiPOD, JUNIWARD, UERD).
+
+# Metric
+
+Weighted AUC. Each region of the ROC curve is weighted according to these chosen parameters:
+
+```
+tpr_thresholds = [0.0, 0.4, 1.0]
+weights = [2, 1]
+```
+
+In other words, the area between the true positive rate of 0 and 0.4 is weighted 2X, the area between 0.4 and 1 is now weighed (1X). The total area is normalized by the sum of weights such that the final weighted AUC is between 0 and 1.
+
+# Submission Format
+
+For each `Id` (image) in the test set, you must provide a score that indicates how likely this image contains hidden data: the higher the score, the more it is assumed that image contains secret data. The file should contain a header and have the following format:
+
+```
+Id,Label
+0001.jpg,0.1
+0002.jpg,0.99
+0003.jpg,1.2
+0004.jpg,-2.2
+etc.
+```
+# Dataset
+
+The only available information on the test set is:
+
+1. Each embedding algorithm is used with the same probability.
+2. The payload (message length) is adjusted such that the "difficulty" is approximately the same regardless the content of the image. Images with smooth content are used to hide shorter messages while highly textured images will be used to hide more secret bits. The payload is adjusted in the same manner for testing and training sets.
+3. The average message length is 0.4 bit per non-zero AC DCT coefficient.
+4. The images are all compressed with one of the three following JPEG quality factors: 95, 90 or 75.
+
+## Files
+
+- `Cover/` contains 75k unaltered images meant for use in training.
+- `JMiPOD/` contains 75k examples of the JMiPOD algorithm applied to the cover images.
+- `JUNIWARD/`contains 75k examples of the JUNIWARD algorithm applied to the cover images.
+- `UERD/` contains 75k examples of the UERD algorithm applied to the cover images.
+- `Test/` contains 5k test set images. These are the images for which you are predicting.
+- `sample_submission.csv` contains an example submission in the correct format.

dslighting/registry/alaska2-image-steganalysis/grade.py

@@ -0,0 +1,136 @@
+import numpy as np
+import pandas as pd
+from sklearn import metrics
+
+from mlebench.grade_helpers import InvalidSubmissionError
+
+
+def insert_thresholds(fpr: np.ndarray, tpr: np.ndarray, tpr_thresholds: list) -> tuple:
+    """
+    Insert tpr_thresholds into the TPR and FPR arrays to ensure that the thresholds are present
+    in the TPR array and the corresponding FPR values are interpolated.
+
+    e.g.
+    > tpr = [0.0, 0.25, 0.5, 0.75, 1.0]
+    > fpr = [0.0, 0.1, 0.2, 0.3, 0.4]
+
+    > tpr_thresholds = [0.0, 0.3, 0.4, 1.0]
+    > fpr, tpr = insert_thresholds(fpr, tpr, tpr_thresholds)
+
+    > print(tpr)
+    > print(fpr)
+    [0.0, 0.25, 0.3, 0.4, 0.5, 0.75, 1.0]
+    [0.0, 0.1, 0.12, 0.16, 0.2, 0.3, 0.4]
+    """
+    fpr_ = fpr.tolist().copy()  # Don't modify the input arrays
+    tpr_ = tpr.tolist().copy()
+    for threshold in tpr_thresholds:
+        if threshold not in tpr_:
+            # Find the right position within tpr to insert the threshold
+            for i, tpr_val in enumerate(tpr_):
+                if tpr_val > threshold:
+                    # Linear interpolation of fpr
+                    new_fpr = fpr_[i - 1] + (fpr_[i] - fpr_[i - 1]) * (threshold - tpr_[i - 1]) / (
+                        tpr_[i] - tpr_[i - 1]
+                    )
+
+                    tpr_.insert(i, threshold)
+                    fpr_.insert(i, new_fpr)
+                    break
+    return np.array(fpr_), np.array(tpr_)
+
+
+def alaska_weighted_auc(y_true: np.ndarray, y_pred: np.ndarray) -> float:
+    """
+    Instead of the standard AUC, the competition uses a weighted AUC where different regions of
+    the ROC curve are weighted differently.
+
+    We compute the area under the curve segment by segment (HORIZONTAL segments between TPR pairs
+    on the y-axis), and then compute a weighted average of the segments.
+
+    For more details, see:
+    www.kaggle.com/competitions/alaska2-image-steganalysis/overview/evaluation
+
+    This particular implementation is adapted from:
+    https://www.kaggle.com/code/anokas/weighted-auc-metric-updated
+    (Key change vs the above implementation: The above implementation uses the `tpr` and `fpr`
+    arrays from metrics.roc_curve as is, neglecting to handle the case where the `tpr` values
+    don't line up nicely with the thresholds - leading to a situation where some segments either
+    belong partially to the wrong threshold, or get skipped entirely. Our implementation fixes
+    this by inserting the thresholds into the `tpr` and `fpr` arrays before computing the AUC.)
+    """
+    tpr_thresholds = [0.0, 0.4, 1.0]
+    weights = [2, 1]
+
+    fpr, tpr, thresholds = metrics.roc_curve(y_true, y_pred, pos_label=1)
+    fpr, tpr = insert_thresholds(fpr, tpr, tpr_thresholds)
+
+    # size of subsets
+    areas = np.array(tpr_thresholds[1:]) - np.array(tpr_thresholds[:-1])
+
+    competition_metric = 0
+    # Compute AUC segment by segment (where each segment is a horizontal slice of the ROC curve
+    # between a pair of consecutive TPR thresholds)
+    for idx, weight in enumerate(weights):
+        y_min = tpr_thresholds[idx]
+        y_max = tpr_thresholds[idx + 1]
+
+        # Here, we're creating new x and y arrays to calculate the AUC for this segment:
+
+        # The segment arrays consist of the FPR and TPR values in the segment,
+        mask = (y_min <= tpr) & (tpr <= y_max)
+        if mask.sum() == 0:
+            continue
+        xs = fpr[mask]
+        ys = tpr[mask]
+
+        # plus a new point [1, y_max] which just closes the shape of this segment (draws a
+        # horizontal line from the highest point in this segment to the right of the x-axis)
+        xs = np.concatenate([xs, [1]])
+        ys = np.concatenate([ys, [y_max]])
+        ys = ys - y_min  # normalize such that curve starts at y=0
+        score = metrics.auc(xs, ys)
+        submetric = score * weight
+        competition_metric += submetric
+
+    # The total area is normalized by the sum of weights such that the final weighted AUC is between 0 and 1.
+    normalization = np.dot(areas, weights)
+
+    return competition_metric / normalization
+
+
+def prepare_for_metric(submission: pd.DataFrame, answers: pd.DataFrame):
+    if len(submission) != len(answers):
+        raise InvalidSubmissionError("Submission and answers must have the same length")
+
+    if "Id" not in submission.columns:
+        raise InvalidSubmissionError("Submission must have an 'Id' column")
+
+    if "Label" not in submission.columns:
+        raise InvalidSubmissionError("Submission must have a 'Label' column")
+
+    submission = submission.sort_values("Id")
+    answers = answers.sort_values("Id")
+
+    if (submission["Id"].values != answers["Id"].values).any():
+        raise InvalidSubmissionError("Submission and answers must have the same ids")
+
+    # Answers and submission must be numbers
+    try:
+        submission["Label"] = submission["Label"].astype(float)
+    except ValueError:
+        raise InvalidSubmissionError("Labels in submission must be numbers")
+    answers["Label"] = answers["Label"].astype(float)
+
+    # Cannot contain NaNs
+    assert not answers["Label"].isnull().any(), "Answers cannot contain NaNs"
+    if submission["Label"].isnull().any():
+        raise InvalidSubmissionError("Submission cannot contain NaNs")
+
+    return answers["Label"], submission["Label"]
+
+
+def grade(submission: pd.DataFrame, answers: pd.DataFrame) -> float:
+    y_true, y_pred = prepare_for_metric(submission, answers)
+
+    return alaska_weighted_auc(np.array(y_true), np.array(y_pred))

dslighting/registry/alaska2-image-steganalysis/prepare.py

@@ -0,0 +1,88 @@
+import random
+import shutil
+from pathlib import Path
+
+import pandas as pd
+from sklearn.model_selection import train_test_split
+from tqdm import tqdm
+
+
+def prepare(raw: Path, public: Path, private: Path):
+    """
+    Splits the data in raw into public and private datasets with appropriate test/train splits.
+    """
+    # List of all train image IDs
+    cover_images_dir = raw / "Cover"
+    cover_images = sorted(list(cover_images_dir.glob("*.jpg")))
+
+    # Create train, test from train split
+    test_size = 5000
+    new_train, new_test = train_test_split(cover_images, test_size=test_size, random_state=42)
+
+    # Prepare dirs
+    steganography_algs = ["JMiPOD", "JUNIWARD", "UERD"]
+    for subdir in ["Cover", "Test"] + steganography_algs:
+        (public / subdir).mkdir(parents=True, exist_ok=True)
+
+    # Copy over new train set, giving them new ids
+    for idx, fp in tqdm(enumerate(new_train), total=len(new_train), desc="Copying train images"):
+        image_id = idx + 1  # 1-indexed
+        shutil.copyfile(src=fp, dst=public / "Cover" / f"{image_id:05d}.jpg")
+        for alg in steganography_algs:
+            shutil.copyfile(src=raw / alg / fp.name, dst=public / alg / f"{image_id:05d}.jpg")
+
+    # Populate test set
+    answers_rows = []
+    random.seed(0)
+    random.shuffle(new_test)
+    for idx, fp in tqdm(enumerate(new_test), total=len(new_test), desc="Copying test images"):
+        image_id = idx + 1  # 1-indexed
+        test_id = f"{image_id:04d}.jpg"
+        dest = public / "Test" / test_id
+
+        # For the test set, we randomly select between the "Cover" (unedited image, negative class)
+        # and one of the 3 steganography algorithms (positive class)
+        # 1:1 ratio of positive:negative examples, and even distribution of steganography algorithms
+        if random.choice([True, False]):
+            # Negative class
+            shutil.copyfile(
+                src=fp,
+                dst=dest,
+            )
+            answers_rows.append({"Id": test_id, "Label": 0})
+        else:
+            # Positive class
+            alg = random.choice(steganography_algs)
+            shutil.copyfile(src=raw / alg / fp.name, dst=dest)
+            answers_rows.append({"Id": test_id, "Label": 1})
+
+    # Write answers to file
+    answers_df = pd.DataFrame(answers_rows)
+    answers_df.to_csv(private / "test.csv", index=False)
+
+    # Create sample submission
+    sample_submission = answers_df.copy()
+    sample_submission["Label"] = 0
+    sample_submission.to_csv(public / "sample_submission.csv", index=False)
+
+    # Checks
+    assert "Id" in answers_df.columns, "Answers must have 'Id' column"
+    assert "Label" in answers_df.columns, "Answers must have 'Label' column"
+    assert "Id" in sample_submission.columns, "Sample submission must have 'Id' column"
+    assert "Label" in sample_submission.columns, "Sample submission must have 'Label' column"
+    assert (
+        len(answers_df) == test_size
+    ), f"Expected {test_size} test images, but got {len(answers_df)}"
+    assert len(sample_submission) == len(
+        answers_df
+    ), f"Sample submission ({len(sample_submission)}) and answers ({len(answers_df)}) must have the same length"
+    assert (
+        len(list(public.glob("Test/*.jpg"))) == test_size
+    ), f"Expected {test_size} test images in public/Test, but got {len(list(public.glob('Test/*.jpg')))}"
+    assert len(list(public.glob("Cover/*.jpg"))) == len(
+        new_train
+    ), f"Expected {len(new_train)} train images in public/Cover, but got {len(list(public.glob('Cover/*.jpg')))}"
+    for alg in steganography_algs:
+        assert len(list(public.glob(f"{alg}/*.jpg"))) == len(
+            new_train
+        ), f"Expected {len(new_train)} train images in public/{alg}, but got {len(list(public.glob(f'{alg}/*.jpg')))}"

dslighting/registry/alaska2-image-steganalysis/prepare_val.py

@@ -0,0 +1,148 @@
+import random
+import shutil
+from pathlib import Path
+
+import pandas as pd
+from sklearn.model_selection import train_test_split
+from tqdm import tqdm
+
+
+def _create_dataset_split(
+    train_image_paths: list,
+    test_image_paths: list,
+    raw_dir: Path,
+    public_dir: Path,
+    private_dir: Path,
+    steganography_algs: list,
+):
+    """
+    Helper function to create a dataset split (e.g., train/test or train_val/test_val).
+
+    This function populates the public and private directories with the respective
+    training images, test images, and test set ground-truth labels.
+    """
+    # Prepare dirs
+    public_dir.mkdir(parents=True, exist_ok=True)
+    private_dir.mkdir(parents=True, exist_ok=True)
+    for subdir in ["Cover", "Test"] + steganography_algs:
+        (public_dir / subdir).mkdir(exist_ok=True)
+
+    # Copy over the train set for this split, giving them new ids
+    for idx, fp in tqdm(
+        enumerate(train_image_paths), total=len(train_image_paths), desc=f"Copying train images to {public_dir.name}"
+    ):
+        image_id = idx + 1  # 1-indexed
+        shutil.copyfile(src=fp, dst=public_dir / "Cover" / f"{image_id:05d}.jpg")
+        for alg in steganography_algs:
+            shutil.copyfile(src=raw_dir / alg / fp.name, dst=public_dir / alg / f"{image_id:05d}.jpg")
+
+    # Populate the test set for this split
+    answers_rows = []
+    random.seed(0)  # Reset seed for deterministic test set creation
+    random.shuffle(test_image_paths)
+    for idx, fp in tqdm(
+        enumerate(test_image_paths), total=len(test_image_paths), desc=f"Copying test images to {public_dir.name}"
+    ):
+        image_id = idx + 1  # 1-indexed
+        test_id = f"{image_id:04d}.jpg"
+        dest = public_dir / "Test" / test_id
+
+        # For the test set, we randomly select between the "Cover" (unedited image, negative class)
+        # and one of the 3 steganography algorithms (positive class)
+        # 1:1 ratio of positive:negative examples, and even distribution of steganography algorithms
+        if random.choice([True, False]):
+            # Negative class
+            shutil.copyfile(
+                src=fp,
+                dst=dest,
+            )
+            answers_rows.append({"Id": test_id, "Label": 0})
+        else:
+            # Positive class
+            alg = random.choice(steganography_algs)
+            shutil.copyfile(src=raw_dir / alg / fp.name, dst=dest)
+            answers_rows.append({"Id": test_id, "Label": 1})
+
+    # Write answers to file
+    answers_df = pd.DataFrame(answers_rows)
+    answers_df.to_csv(private_dir / "test.csv", index=False)
+
+    # Create sample submission
+    sample_submission = answers_df.copy()
+    sample_submission["Label"] = 0
+    sample_submission.to_csv(public_dir / "sample_submission.csv", index=False)
+
+    # Checks
+    test_size = len(test_image_paths)
+    assert "Id" in answers_df.columns, "Answers must have 'Id' column"
+    assert "Label" in answers_df.columns, "Answers must have 'Label' column"
+    assert "Id" in sample_submission.columns, "Sample submission must have 'Id' column"
+    assert "Label" in sample_submission.columns, "Sample submission must have 'Label' column"
+    assert (
+        len(answers_df) == test_size
+    ), f"Expected {test_size} test images, but got {len(answers_df)}"
+    assert len(sample_submission) == len(
+        answers_df
+    ), f"Sample submission ({len(sample_submission)}) and answers ({len(answers_df)}) must have the same length"
+    assert (
+        len(list(public_dir.glob("Test/*.jpg"))) == test_size
+    ), f"Expected {test_size} test images in {public_dir.name}/Test, but got {len(list(public_dir.glob('Test/*.jpg')))}"
+    assert len(list(public_dir.glob("Cover/*.jpg"))) == len(
+        train_image_paths
+    ), f"Expected {len(train_image_paths)} train images in {public_dir.name}/Cover, but got {len(list(public_dir.glob('Cover/*.jpg')))}"
+    for alg in steganography_algs:
+        assert len(list(public_dir.glob(f"{alg}/*.jpg"))) == len(
+            train_image_paths
+        ), f"Expected {len(train_image_paths)} train images in {public_dir.name}/{alg}, but got {len(list(public_dir.glob(f'{alg}/*.jpg')))}"
+
+
+def prepare(raw: Path, public: Path, private: Path):
+    """
+    Splits the data in raw into public and private datasets with appropriate test/train splits.
+    Also creates a secondary validation split (train_val/test_val) in parallel directories.
+    """
+    # List of all train image IDs
+    cover_images_dir = raw / "Cover"
+    cover_images = sorted(list(cover_images_dir.glob("*.jpg")))
+    steganography_algs = ["JMiPOD", "JUNIWARD", "UERD"]
+    test_size = 5000
+
+    # --- Stage 1: Create the main competition train/test split ---
+    # This split creates the final test set used for scoring.
+    # The outputs in `public` and `private` are left untouched by subsequent steps.
+    train_main, test_main = train_test_split(
+        cover_images, test_size=test_size, random_state=42
+    )
+
+    _create_dataset_split(
+        train_image_paths=train_main,
+        test_image_paths=test_main,
+        raw_dir=raw,
+        public_dir=public,
+        private_dir=private,
+        steganography_algs=steganography_algs,
+    )
+
+    # --- Stage 2: Create the validation train/test split ---
+    # This performs a second split on the main training data (`train_main`)
+    # to create a new, smaller training set and a validation set.
+    # Outputs are saved to `public_val` and `private_val` to avoid conflicts.
+    public_val = public.parent / "public_val"
+    private_val = private.parent / "private_val"
+
+    # Split `train_main` again, using the same logic as the first split
+    # to create a validation set of the same size as the main test set.
+    train_val, test_val = train_test_split(
+        train_main, test_size=test_size, random_state=42
+    )
+
+    # The new split is processed using the same helper function to ensure
+    # identical directory structure, filenames, and creation logic.
+    _create_dataset_split(
+        train_image_paths=train_val,
+        test_image_paths=test_val,
+        raw_dir=raw,
+        public_dir=public_val,
+        private_dir=private_val,
+        steganography_algs=steganography_algs,
+    )