dslighting 1.3.9__py3-none-any.whl → 1.7.6__py3-none-any.whl

dslighting/registry/nomad2018-predict-transparent-conductors/prepare_val.py

@@ -0,0 +1,144 @@
+import glob
+import shutil
+from pathlib import Path
+
+import pandas as pd
+from sklearn.model_selection import train_test_split
+
+from mlebench.utils import extract, read_csv
+
+
+def _process_split(
+    source_df: pd.DataFrame,
+    test_size: float,
+    random_state: int,
+    source_geometry_paths: list,
+    public_dir: Path,
+    private_dir: Path,
+) -> pd.DataFrame:
+    """
+    Helper function to perform a data split, re-index, and write all necessary files.
+
+    Args:
+        source_df: The DataFrame to split.
+        test_size: The proportion of the dataset to allocate to the test split.
+        random_state: The seed used by the random number generator.
+        source_geometry_paths: A list of paths to all original geometry files.
+        public_dir: The destination directory for public-facing files (train set, test features).
+        private_dir: The destination directory for private-facing files (test labels).
+
+    Returns:
+        The created training DataFrame, which can be used for a subsequent split.
+    """
+    # Ensure destination directories exist
+    public_dir.mkdir(parents=True, exist_ok=True)
+    private_dir.mkdir(parents=True, exist_ok=True)
+
+    # Create train, test from the source dataframe
+    new_train, new_test = train_test_split(
+        source_df, test_size=test_size, random_state=random_state
+    )
+
+    # Make ids go 1, 2, ... for both train and test. Keep old ids so we can map ids of other files
+    old_train_id_to_new = {
+        old_id: new_id for new_id, old_id in enumerate(new_train["id"], start=1)
+    }  # id starts from 1
+    new_train["id"] = new_train["id"].map(old_train_id_to_new)
+
+    old_test_id_to_new = {
+        old_id: new_id for new_id, old_id in enumerate(new_test["id"], start=1)
+    }  # id starts from 1
+    new_test["id"] = new_test["id"].map(old_test_id_to_new)
+
+    new_test_without_labels = new_test.drop(
+        columns=["formation_energy_ev_natom", "bandgap_energy_ev"]
+    )
+
+    # Copy over files
+    new_train.to_csv(public_dir / "train.csv", index=False)
+    new_test.to_csv(private_dir / "test.csv", index=False)
+    new_test_without_labels.to_csv(public_dir / "test.csv", index=False)
+
+    # --- Process and copy geometry files for the new train set ---
+    train_geometry_dir = public_dir / "train"
+    for src in source_geometry_paths:
+        original_id = int(Path(src).parts[-2])
+        if original_id not in old_train_id_to_new:  # Filter for train ids
+            continue
+
+        new_id = old_train_id_to_new[original_id]
+        dest_dir = train_geometry_dir / str(new_id)
+        dest_dir.mkdir(parents=True, exist_ok=True)
+        shutil.copy(src=src, dst=dest_dir / "geometry.xyz")
+    assert len(list(train_geometry_dir.glob("**/*.xyz"))) == len(
+        new_train
+    ), f"Expected {len(new_train)} train geometry files in {public_dir}, found {len(list(train_geometry_dir.glob('**/*.xyz')))}"
+
+    # --- Process and copy geometry files for the new test set ---
+    test_geometry_dir = public_dir / "test"
+    for src in source_geometry_paths:
+        original_id = int(Path(src).parts[-2])
+        if original_id not in old_test_id_to_new:  # Filter for test ids
+            continue
+
+        new_id = old_test_id_to_new[original_id]
+        dest_dir = test_geometry_dir / str(new_id)
+        dest_dir.mkdir(parents=True, exist_ok=True)
+        shutil.copy(src=src, dst=dest_dir / "geometry.xyz")
+    assert len(list(test_geometry_dir.glob("**/*.xyz"))) == len(
+        new_test
+    ), f"Expected {len(new_test)} test geometry files in {public_dir}, found {len(list(test_geometry_dir.glob('**/*.xyz')))}"
+
+    # Create mock submission
+    sample_submission = pd.DataFrame(
+        {"id": new_test["id"], "formation_energy_ev_natom": 0.1779, "bandgap_energy_ev": 1.8892}
+    )
+    sample_submission.to_csv(public_dir / "sample_submission.csv", index=False)
+    assert len(sample_submission) == len(
+        new_test
+    ), f"Sample submission in {public_dir} should have the same number of rows as its test set"
+
+    return new_train
+
+
+def prepare(raw: Path, public: Path, private: Path):
+    """
+    Splits the data in raw into public and private datasets with appropriate test/train splits.
+    Also creates a secondary validation split (public_val/private_val) from the training data.
+    """
+    # Define paths for the new validation set
+    public_val = public.parent / "public_val"
+    private_val = private.parent / "private_val"
+
+    # Extract only what we need
+    extract(raw / "train.zip", raw / "train")
+    extract(raw / "train.csv.zip", raw / "train.csv")
+    extract(raw / "test.zip", raw / "test")
+    extract(raw / "test.csv.zip", raw / "test.csv")
+
+    # Load initial data and geometry paths
+    initial_train_df = read_csv(raw / "train.csv/train.csv")
+    all_geometry_paths = sorted(glob.glob(str(raw / "train/train/**/*.xyz")))
+
+    # --- First Split: Create original public/private datasets ---
+    # This split is identical to the original script to ensure unchanged outputs.
+    train_df_after_first_split = _process_split(
+        source_df=initial_train_df,
+        test_size=0.1,
+        random_state=0,
+        source_geometry_paths=all_geometry_paths,
+        public_dir=public,
+        private_dir=private,
+    )
+
+    # --- Second Split: Create new validation datasets from the first split's train set ---
+    # The test_size is set to 1/9 of the incoming data (which is 90% of the original).
+    # This makes the new test_val set size equal to the original test set size (0.1 * T = 1/9 * 0.9 * T).
+    _process_split(
+        source_df=train_df_after_first_split,
+        test_size=1 / 9,
+        random_state=0,
+        source_geometry_paths=all_geometry_paths,
+        public_dir=public_val,
+        private_dir=private_val,
+    )

dslighting/registry/osic-pulmonary-fibrosis-progression/checksums.yaml

@@ -0,0 +1,7 @@
+private:
+  test.csv: 0487bd97c387b5e6c28d5060026462d9
+public:
+  sample_submission.csv: 6204ba4dd60cbf2629cd43c60144b9a7
+  test.csv: 4d8bf1fd3b55feae1a1e3b5088a2f268
+  train.csv: dd984a7b700fcd6f167c89ebbbe8619e
+zip: c158d5991c84fa57940d8be0eb53272f

dslighting/registry/osic-pulmonary-fibrosis-progression/config.yaml

@@ -0,0 +1,22 @@
+id: osic-pulmonary-fibrosis-progression
+name: OSIC Pulmonary Fibrosis Progression
+competition_type: code
+awards_medals: true
+prizes:
+  - position: 1
+    value: 30000
+  - position: 2
+    value: 15000
+  - position: 3
+    value: 10000
+description: mlebench/competitions/osic-pulmonary-fibrosis-progression/description.md
+
+dataset:
+  answers: osic-pulmonary-fibrosis-progression/prepared/private/test.csv
+  sample_submission: osic-pulmonary-fibrosis-progression/prepared/public/sample_submission.csv
+
+grader:
+  name: modified-laplace-log-likelihood
+  grade_fn: mlebench.competitions.osic-pulmonary-fibrosis-progression.grade:grade
+
+preparer: mlebench.competitions.osic-pulmonary-fibrosis-progression.prepare:prepare

dslighting/registry/osic-pulmonary-fibrosis-progression/description.md

@@ -0,0 +1,122 @@
+## **Description**
+
+Imagine one day, your breathing became consistently labored and shallow. Months later you were finally diagnosed with pulmonary fibrosis, a disorder with no known cause and no known cure, created by scarring of the lungs. If that happened to you, you would want to know your prognosis. That’s where a troubling disease becomes frightening for the patient: outcomes can range from long-term stability to rapid deterioration, but doctors aren’t easily able to tell where an individual may fall on that spectrum. Your help, and data science, may be able to aid in this prediction, which would dramatically help both patients and clinicians.
+
+![Untitled](https://prod-files-secure.s3.us-west-2.amazonaws.com/667f1cbf-826f-4641-a321-96054292638d/d4ebeea7-3764-4f7b-b4f7-35b2d0ecbeea/Untitled.png)
+
+Current methods make fibrotic lung diseases difficult to treat, even with access to a chest CT scan. In addition, the wide range of varied prognoses create issues organizing clinical trials. Finally, patients suffer extreme anxiety—in addition to fibrosis-related symptoms—from the disease’s opaque path of progression.
+
+[Open Source Imaging Consortium (OSIC)](https://www.osicild.org/) is a not-for-profit, co-operative effort between academia, industry and philanthropy. The group enables rapid advances in the fight against Idiopathic Pulmonary Fibrosis (IPF), fibrosing interstitial lung diseases (ILDs), and other respiratory diseases, including emphysematous conditions. Its mission is to bring together radiologists, clinicians and computational scientists from around the world to improve imaging-based treatments.
+
+In this competition, you’ll predict a patient’s severity of decline in lung function based on a CT scan of their lungs. You’ll determine lung function based on output from a spirometer, which measures the volume of air inhaled and exhaled. The challenge is to use machine learning techniques to make a prediction with the image, metadata, and baseline FVC as input.
+
+If successful, patients and their families would better understand their prognosis when they are first diagnosed with this incurable lung disease. Improved severity detection would also positively impact treatment trial design and accelerate the clinical development of novel treatments.
+
+**This is a Code Competition. Refer to [Code Requirements](https://www.kaggle.com/c/osic-pulmonary-fibrosis-progression#Code-Requirements) for details.**
+
+## **Evaluation**
+
+This competition is evaluated on a modified version of the Laplace Log Likelihood. In medical applications, it is useful to evaluate a model's confidence in its decisions. Accordingly, the metric is designed to reflect both the accuracy and certainty of each prediction.
+
+For each true FVC measurement, you will predict both an FVC and a confidence measure (standard deviation 𝜎𝜎). The metric is computed as:
+
+$$
+\begin{gathered}
+\sigma_{\text {clipped }}=\max (\sigma, 70), \\
+\Delta=\min \left(\left|F V C_{\text {true }}-F V C_{\text {predicted }}\right|, 1000\right), \\
+\text { metric }=-\frac{\sqrt{2} \Delta}{\sigma_{\text {clipped }}}-\ln \left(\sqrt{2} \sigma_{\text {clipped }}\right) .
+\end{gathered}
+$$
+
+The error is thresholded at 1000 ml to avoid large errors adversely penalizing results, while the confidence values are clipped at 70 ml to reflect the approximate measurement uncertainty in FVC. The final score is calculated by averaging the metric across all test set `Patient_Week`s (three per patient). Note that metric values will be negative and higher is better.
+
+## **Submission File**
+
+For each `Patient_Week`, you must predict the `FVC` and a confidence. To avoid potential leakage in the timing of follow up visits, you are asked to predict every patient's `FVC` measurement for every possible week. Those weeks which are not in the final three visits are ignored in scoring.
+
+The file should contain a header and have the following format:
+
+```
+Patient_Week,FVC,Confidence
+ID00002637202176704235138_1,2000,100
+ID00002637202176704235138_2,2000,100
+ID00002637202176704235138_3,2000,100
+etc.
+
+```
+
+## **Timeline**
+
+- **September 29, 2020** - Entry deadline. You must accept the competition rules before this date in order to compete.
+- **September 29, 2020** - Team Merger deadline. This is the last day participants may join or merge teams.
+- **October 6, 2020** - Final submission deadline.
+
+All deadlines are at 11:59 PM UTC on the corresponding day unless otherwise noted. The competition organizers reserve the right to update the contest timeline if they deem it necessary.
+
+## **Prizes**
+
+- 1st Place - $30,000
+- 2nd Place - $15,000
+- 3rd Place - $10,000
+
+## **Code Requirements**
+
+### **This is a Code Competition**
+
+Submissions to this competition must be made through Notebooks. In order for the "Submit to Competition" button to be active after a commit, the following conditions must be met:
+
+- CPU Notebook <= 9 hours run-time
+- GPU Notebook <= 4 hours run-time
+- [TPUs](https://www.kaggle.com/docs/tpu) will not be available for making submissions to this competition. You are still welcome to use them for training models.
+- No internet access enabled
+- External data, freely & publicly available, is allowed. This includes pre-trained models.
+- Submission file must be named `submission.csv`
+
+Please see the [Code Competition FAQ](https://www.kaggle.com/docs/competitions#notebooks-only-FAQ) for more information on how to submit.
+
+## **Citation**
+
+Ahmed Shahin, Carmela Wegworth, David, Elizabeth Estes, Julia Elliott, Justin Zita, SimonWalsh, Slepetys, Will Cukierski. (2020). OSIC Pulmonary Fibrosis Progression. Kaggle. https://kaggle.com/competitions/osic-pulmonary-fibrosis-progression
+
+# Data
+
+## **Dataset Description**
+
+The aim of this competition is to predict a patient’s severity of decline in lung function based on a CT scan of their lungs. Lung function is assessed based on output from a spirometer, which measures the forced vital capacity (`FVC`), i.e. the volume of air exhaled.
+
+In the dataset, you are provided with a baseline chest CT scan and associated clinical information for a set of patients. A patient has an image acquired at time `Week = 0` and has numerous follow up visits over the course of approximately 1-2 years, at which time their `FVC` is measured.
+
+- In the training set, you are provided with an anonymized, baseline CT scan and the entire history of FVC measurements.
+- In the test set, you are provided with a baseline CT scan and only the initial FVC measurement. **You are asked to predict the final three `FVC` measurements for each patient, as well as a confidence value in your prediction.**
+
+There are around 200 cases in the public & private test sets, combined. This is split roughly 15-85 between public-private.
+
+Since this is real medical data, you will notice the relative timing of `FVC` measurements varies widely. The timing of the initial measurement relative to the CT scan and the duration to the forecasted time points may be different for each patient. This is considered part of the challenge of the competition. To avoid potential leakage in the timing of follow up visits, you are asked to predict every patient's `FVC` measurement for every possible week. Those weeks which are not in the final three visits are ignored in scoring.
+
+### **Files**
+
+This is a synchronous rerun code competition. The provided test set is a small representative set of files (copied from the training set) to demonstrate the format of the private test set. When you submit your notebook, Kaggle will rerun your code on the test set, which contains unseen images.
+
+- **train.csv** - the training set, contains full history of clinical information
+- **test.csv** - the test set, contains only the baseline measurement
+- **train/** - contains the training patients' baseline CT scan in DICOM format
+- **test/** - contains the test patients' baseline CT scan in DICOM format
+- **sample_submission.csv** - demonstrates the submission format
+
+### **Columns**
+
+**train.csv and test.csv**
+
+- `Patient`a unique Id for each patient (also the name of the patient's DICOM folder)
+- `Weeks`the relative number of weeks pre/post the baseline CT (may be negative)
+- `FVC` - the recorded lung capacity in ml
+- `Percent`a computed field which approximates the patient's FVC as a percent of the typical FVC for a person of similar characteristics
+- `Age`
+- `Sex`
+- `SmokingStatus`
+
+**sample submission.csv**
+
+- `Patient_Week` - a unique Id formed by concatenating the `Patient` and `Weeks` columns (i.e. ABC_22 is a prediction for patient ABC at week 22)
+- `FVC` - the predicted FVC in ml
+- `Confidence` - a confidence value of your prediction (also has units of ml)

dslighting/registry/osic-pulmonary-fibrosis-progression/description_obfuscated.md

@@ -0,0 +1,65 @@
+# Task
+
+Predict a patient’s severity of decline in lung function based on a CT scan of their lungs. Lung function is assessed based on output from a spirometer, which measures the forced vital capacity (`FVC`), i.e. the volume of air exhaled.
+
+# Metric
+
+A modified version of the Laplace Log Likelihood. 
+
+For each true FVC measurement, you will predict both an FVC and a confidence measure (standard deviation 𝜎𝜎). The metric is computed as:
+
+$$
+\begin{gathered}
+\sigma_{\text {clipped }}=\max (\sigma, 70), \\
+\Delta=\min \left(\left|F V C_{\text {true }}-F V C_{\text {predicted }}\right|, 1000\right), \\
+\text { metric }=-\frac{\sqrt{2} \Delta}{\sigma_{\text {clipped }}}-\ln \left(\sqrt{2} \sigma_{\text {clipped }}\right) .
+\end{gathered}
+$$
+
+The error is thresholded at 1000 ml to avoid large errors adversely penalizing results, while the confidence values are clipped at 70 ml to reflect the approximate measurement uncertainty in FVC. The final score is calculated by averaging the metric across all test set `Patient_Week`s (three per patient). 
+
+Metric values will be negative and higher is better.
+
+# Submission Format
+
+For each `Patient_Week`, you must predict the `FVC` and a confidence. You are asked to predict every patient's `FVC` measurement for every possible week. Those weeks which are not in the final three visits are ignored in scoring.
+
+The file should contain a header and have the following format:
+
+```
+Patient_Week,FVC,Confidence
+ID00002637202176704235138_1,2000,100
+ID00002637202176704235138_2,2000,100
+ID00002637202176704235138_3,2000,100
+etc.
+
+```
+
+# Dataset
+
+In the dataset, you are provided with a baseline chest CT scan and associated clinical information for a set of patients. A patient has an image acquired at time `Week = 0` and has numerous follow up visits over the course of approximately 1-2 years, at which time their `FVC` is measured.
+
+- In the training set, you are provided with an anonymized, baseline CT scan and the entire history of FVC measurements.
+- In the test set, you are provided with a baseline CT scan and only the initial FVC measurement. **You are asked to predict the final three `FVC` measurements for each patient, as well as a confidence value in your prediction.**
+
+- **train.csv** - the training set, contains full history of clinical information
+- **test.csv** - the test set, contains only the baseline measurement
+- **train/** - contains the training patients' baseline CT scan in DICOM format
+- **test/** - contains the test patients' baseline CT scan in DICOM format
+- **sample_submission.csv** - demonstrates the submission format
+
+**train.csv and test.csv**
+
+- `Patient`a unique Id for each patient (also the name of the patient's DICOM folder)
+- `Weeks`the relative number of weeks pre/post the baseline CT (may be negative)
+- `FVC` - the recorded lung capacity in ml
+- `Percent`a computed field which approximates the patient's FVC as a percent of the typical FVC for a person of similar characteristics
+- `Age`
+- `Sex`
+- `SmokingStatus`
+
+**sample submission.csv**
+
+- `Patient_Week` - a unique Id formed by concatenating the `Patient` and `Weeks` columns (i.e. ABC_22 is a prediction for patient ABC at week 22)
+- `FVC` - the predicted FVC in ml
+- `Confidence` - a confidence value of your prediction (also has units of ml)

dslighting/registry/osic-pulmonary-fibrosis-progression/grade.py

@@ -0,0 +1,74 @@
+import numpy as np
+import pandas as pd
+from pandas import DataFrame
+
+from mlebench.grade_helpers import InvalidSubmissionError
+
+
+def prepare_for_metric(submission: DataFrame, answers: DataFrame) -> dict:
+
+    assert "Patient_Week" in answers.columns, "Answers DataFrame must have a 'Patient_Week' column."
+    assert "FVC" in answers.columns, "Answers DataFrame must have a 'FVC' column."
+    assert "Patient" in answers.columns, "Answers DataFrame must have a 'Patient' column."
+    if "Patient_Week" not in submission.columns:
+        raise InvalidSubmissionError("Submission DataFrame must have a 'Patient_Week' column.")
+    if "FVC" not in submission.columns:
+        raise InvalidSubmissionError("Submission DataFrame must have a 'FVC' column.")
+    if "Confidence" not in submission.columns:
+        raise InvalidSubmissionError("Submission DataFrame must have a 'Confidence' column.")
+    for pw in submission["Patient_Week"]:
+        if pw not in answers["Patient_Week"].values:
+            raise InvalidSubmissionError(
+                f"Patient_Week {pw} in submission does not exist in answers"
+            )
+    if not pd.api.types.is_numeric_dtype(submission["FVC"]):
+        raise InvalidSubmissionError("FVC column in submission must be numeric.")
+    if not pd.api.types.is_numeric_dtype(submission["Confidence"]):
+        raise InvalidSubmissionError("Confidence column in submission must be numeric.")
+
+    # fillna with 0 for the confidence column
+    submission["Confidence"] = submission["Confidence"].fillna(0)
+
+    # We should only take the 3 latest Patient_Week records for each patient
+    answers = answers.dropna(subset=["FVC"])  # Drop the dummy data
+    answers["Week_Number"] = answers["Patient_Week"].apply(lambda x: int(x.split("_")[-1]))
+    latest_weeks = answers.sort_values("Week_Number").groupby("Patient").tail(3)
+    answers = latest_weeks.drop(columns=["Week_Number"])
+    # Make submission match; we only grade the prediction for the 3 latest weeks
+    submission = submission[submission["Patient_Week"].isin(answers["Patient_Week"])]
+
+    submission = submission.sort_values(by="Patient_Week")
+    answers = answers.sort_values(by="Patient_Week")
+
+    fvc_true = answers.loc[answers["Patient_Week"].isin(submission["Patient_Week"]), "FVC"].values
+    fvc_pred = submission.loc[
+        submission["Patient_Week"].isin(answers["Patient_Week"]), "FVC"
+    ].values
+    confidence = submission.loc[
+        submission["Patient_Week"].isin(answers["Patient_Week"]), "Confidence"
+    ].values
+
+    return {"fvc_true": fvc_true, "fvc_pred": fvc_pred, "confidence": confidence}
+
+
+def laplace_log_likelihood(
+    actual_fvc: np.ndarray, predicted_fvc: np.ndarray, confidence: np.ndarray, return_values=False
+) -> float:
+    """
+    Calculates the modified Laplace Log Likelihood score for osic-pulmonary-fibrosis-progression
+    See https://www.kaggle.com/code/rohanrao/osic-understanding-laplace-log-likelihood for details on this implementation
+    """
+    sd_clipped = np.maximum(confidence, 70)
+    delta = np.minimum(np.abs(actual_fvc - predicted_fvc), 1000)
+    metric = -np.sqrt(2) * delta / sd_clipped - np.log(np.sqrt(2) * sd_clipped)
+
+    if return_values:
+        return metric
+    else:
+        return np.mean(metric)
+
+
+def grade(submission: DataFrame, answers: DataFrame) -> float:
+    fvc_true, fvc_pred, confidence = prepare_for_metric(submission, answers).values()
+
+    return laplace_log_likelihood(fvc_true, fvc_pred, confidence)

dslighting/registry/osic-pulmonary-fibrosis-progression/prepare.py

@@ -0,0 +1,95 @@
+import shutil
+from multiprocessing import Pool
+from pathlib import Path
+
+import pandas as pd
+from sklearn.model_selection import train_test_split
+from tqdm import tqdm
+
+from mlebench.utils import read_csv
+
+
+def copy_dir(args):
+    src_dir, dst_dir = args
+    shutil.copytree(src=src_dir, dst=dst_dir, dirs_exist_ok=True)
+
+
+def prepare(raw: Path, public: Path, private: Path):
+    # Splitting the train set into train and test with unique Patients
+    old_train = read_csv(raw / "train.csv")
+    grouped_by_patient = list(old_train.groupby("Patient"))
+    train_groups, test_groups = train_test_split(grouped_by_patient, test_size=0.1, random_state=0)
+    new_train = pd.concat([group for _, group in train_groups])
+    new_test = pd.concat([group for _, group in test_groups])
+    assert set(new_train["Patient"]).isdisjoint(
+        set(new_test["Patient"])
+    ), "There are Patients who are in both train and test sets."
+
+    # For the public new_test set we will only keep each patients first FVS measurement. The task is to predict FVS measurements for all possible weeks
+    new_test_public = new_test.sort_values(by="Weeks").groupby("Patient").first().reset_index()
+
+    # Creating the private answers CSV. We need to fill out dummy FVS measurements for all weeks that don't have data so as to match sample_submission.csv
+    # Create a DataFrame with all possible Patient-Week combinations
+    all_weeks = pd.DataFrame(
+        [
+            (patient, week)
+            for patient in new_test["Patient"].unique()
+            for week in range(new_test["Weeks"].min(), new_test["Weeks"].max() + 1)
+        ],
+        columns=["Patient", "Weeks"],
+    )
+    # Merge with the new_test DataFrame to fill in missing weeks with NaN values
+    new_test = all_weeks.merge(new_test, on=["Patient", "Weeks"], how="left")
+    new_test["Patient_Week"] = new_test["Patient"] + "_" + new_test["Weeks"].astype(str)
+    new_test["Confidence"] = 100
+    assert (
+        new_test.groupby("Patient").size().nunique() == 1
+    ), "Not all patients have the same number of rows."
+
+    # Create a sample submission file
+    submission_df = new_test.copy()
+    submission_df = submission_df[["Patient_Week"]]
+    submission_df["FVC"] = 2000  # Dummy predictions
+    submission_df["Confidence"] = 100  # Dummy confidence
+
+    # Write CSVs
+    new_train.to_csv(public / "train.csv", index=False)
+    new_test_public.to_csv(public / "test.csv", index=False)
+    new_test.to_csv(private / "test.csv", index=False)
+    submission_df.to_csv(public / "sample_submission.csv", index=False)
+
+    # Copy over data files
+    (public / "train").mkdir(exist_ok=True)
+    train_args = [
+        (raw / "train" / patient, public / "train" / patient)
+        for patient in new_train["Patient"].unique()
+    ]
+    with Pool() as pool:
+        list(
+            tqdm(pool.imap(copy_dir, train_args), total=len(train_args), desc="Copying train data")
+        )
+
+    (public / "test").mkdir(exist_ok=True)
+    test_args = [
+        (raw / "train" / patient, public / "test" / patient)
+        for patient in new_test["Patient"].unique()
+    ]
+    with Pool() as pool:
+        list(tqdm(pool.imap(copy_dir, test_args), total=len(test_args), desc="Copying test data"))
+
+    # Final checks
+    assert new_train.shape[1] == 7, f"Expected 7 columns in new_train, but got {new_train.shape[1]}"
+    assert new_test.shape[1] == 9, f"Expected 9 columns in new_test, but got {new_test.shape[1]}"
+    assert (
+        new_test_public.shape[1] == 7
+    ), f"Expected 7 columns in new_test_public, but got {new_test_public.shape[1]}"
+    assert (
+        submission_df.shape[1] == 3
+    ), f"Expected 3 columns in submission_df, but got {submission_df.shape[1]}"
+
+    public_train_dirs = set((public / "train").iterdir())
+    public_test_dirs = set((public / "test").iterdir())
+    common_dirs = public_train_dirs.intersection(public_test_dirs)
+    assert (
+        not common_dirs
+    ), f"There are directories with the same name in public train and test: {common_dirs}"