dslighting 1.3.9__py3-none-any.whl → 1.7.6__py3-none-any.whl

dslighting/registry/siim-covid19-detection/grade.py

@@ -0,0 +1,194 @@
+from typing import List, Tuple
+
+import numpy as np
+import pandas as pd
+from sklearn.metrics import average_precision_score
+
+from mlebench.grade_helpers import InvalidSubmissionError
+
+
+def calculate_iou(
+    box1: Tuple[float, float, float, float], box2: Tuple[float, float, float, float]
+) -> float:
+    """
+    Calculate the Intersection over Union (IoU) of two bounding boxes.
+    (this image helps: https://machinelearningspace.com/wp-content/uploads/2023/01/IOU2.jpg)
+    """
+    xmin1, ymin1, xmax1, ymax1 = box1
+    xmin2, ymin2, xmax2, ymax2 = box2
+
+    inter_xmin = max(xmin1, xmin2)
+    inter_ymin = max(ymin1, ymin2)
+    inter_xmax = min(xmax1, xmax2)
+    inter_ymax = min(ymax1, ymax2)
+
+    inter_width = max(0, inter_xmax - inter_xmin)
+    inter_height = max(0, inter_ymax - inter_ymin)
+    inter_area = inter_width * inter_height
+
+    box1_area = (xmax1 - xmin1) * (ymax1 - ymin1)
+    box2_area = (xmax2 - xmin2) * (ymax2 - ymin2)
+
+    union_area = box1_area + box2_area - inter_area
+
+    iou = inter_area / union_area if union_area > 0 else 0.0
+    return iou
+
+
+def calculate_map(submission_preds: List[List[Tuple]], answers_preds: List[List[Tuple]]) -> float:
+    """
+    Calculate mean Average Precision (mAP) for object detection.
+
+    >   The challenge uses the standard PASCAL VOC 2010 mean Average Precision (mAP) at IoU > 0.5.
+        Note that the linked document describes VOC 2012, which differs in some minor ways
+        (e.g. there is no concept of "difficult" classes in VOC 2010).
+        The P/R curve and AP calculations remain the same.
+    (www.kaggle.com/competitions/siim-covid19-detection/overview/evaluation)
+
+    Some choices made here that were not explicitly specified in the challenge description:
+
+    1. Treating "none" or "negative" as prediction classes of their own, instead of as the non-positive class
+    -   Justification: Treating them as their own classes is implied by the data format:
+        - Study level - train_study_level.csv has 4 binary classes including the "negative" class instead of 3
+        - Image level - We're asked to predict "none" with full bounding boxes instead of withholding a prediction
+        Also, 3rd place winner says "It probably treats the six classes equally", where the six classes are
+        "negative", "typical", "indeterminate", "atypical", "none" and "opacity".
+        (https://www.kaggle.com/competitions/siim-covid19-detection/discussion/240363)
+
+    2. Rules for populating y_pairs (see comments below), in particular the (0, 0) case
+    -   Justification: The general rules follow the descriptions of the PASCAL VOC 2010 mAP documented online.
+        The only custom addition is handling the edge case of (0, 0) false negatives, which was necessary because
+        if we don't include (0, 0) pairs, both the sample submission and gold submission end up with all values of
+        y_true being 1, so the AP is undefined.
+        Behavior of our implementation is consistent with this comment from the organizers:
+        https://www.kaggle.com/competitions/siim-covid19-detection/discussion/248467#1362916
+
+    """
+    aps = []
+
+    # Group predictions by class - the general idea is to calculate AP for each class separately
+    # and then average them to get mAP
+    classes = sorted(list(set(pred[0] for preds in answers_preds for pred in preds)))
+
+    for cls in classes:
+        # We will populate y_true and y_scores with (y_true, y_score) of:
+        # (1, confidence) for every predicted box that matches a ground truth box
+        # (0, confidence) for every predicted box that does not match a ground truth box
+        # (1, 0) for every ground truth box that is not matched to a predicted box
+        # (0, 0) when there are neither predicted nor ground truth boxes
+        y_pairs = []  # List of (y_true, y_score) pairs
+
+        # Gather all predictions and ground truth boxes related to this class from all samples
+        for img_preds, img_gts in zip(submission_preds, answers_preds):
+            y_pairs_ = []
+
+            # Get ground truth boxes for this class
+            gt_boxes = [gt[2:] for gt in img_gts if gt[0] == cls]
+
+            # Sort img_preds by confidence
+            img_preds.sort(key=lambda x: x[1], reverse=True)
+
+            # For each prediction of this class
+            matched_gt = [False] * len(gt_boxes)  # Initialize all ground truths as unmatched
+            for pred in img_preds:
+                if pred[0] == cls:
+                    # Find the best matching ground truth box
+                    best_iou = 0
+                    best_gt_idx = -1
+                    for i, gt in enumerate(gt_boxes):
+                        if matched_gt[i]:  # Don't reuse matched ground truths
+                            continue
+                        iou = calculate_iou(pred[2:], gt)
+                        if iou > best_iou and iou > 0.5:
+                            best_iou = iou
+                            best_gt_idx = i
+
+                    pred_confidence = pred[1]
+                    if best_gt_idx != -1:
+                        y_pairs_.append((1, pred_confidence))  # True positive
+                        matched_gt[best_gt_idx] = True
+                    else:
+                        y_pairs_.append((0, pred_confidence))  # False positive
+
+            # Add false negatives for unmatched ground truths
+            y_pairs_.extend([(1, 0)] * matched_gt.count(False))
+
+            if len(y_pairs_) == 0:
+                # A true negative
+                y_pairs_.append((0, 0))
+
+            y_pairs.extend(y_pairs_)
+
+        y_true = [pair[0] for pair in y_pairs]
+        y_scores = [pair[1] for pair in y_pairs]
+        if len(y_true) > 0:
+            assert not all(
+                y == 1 for y in y_true
+            ), "y_true is all 1s; this shouldn't happen and will result in undefined AP"
+            ap = average_precision_score(y_true, y_scores)
+            aps.append(ap)
+        else:
+            raise ValueError(f"Unexpected error: y_true is empty for class {cls}")
+
+    # Calculate mAP
+    return np.mean(aps) if aps else 0.0
+
+
+def parse_prediction_string(
+    prediction_string: str,
+) -> List[Tuple[str, float, float, float, float, float]]:
+    """
+    `prediction_string` should have the form "{label} {confidence} {xmin} {ymin} {xmax} {ymax}" for each bounding box
+    (so there may be 6 x N space-separated values in total, where N is the number of bounding boxes).
+
+    Returns a list of tuples, each with the form (label, confidence, xmin, ymin, xmax, ymax).
+    """
+    toks = prediction_string.strip().split()
+    try:
+        return [
+            (
+                toks[i],
+                float(toks[i + 1]),
+                float(toks[i + 2]),
+                float(toks[i + 3]),
+                float(toks[i + 4]),
+                float(toks[i + 5]),
+            )
+            for i in range(0, len(toks), 6)
+        ]
+    except ValueError as e:
+        raise InvalidSubmissionError(f"Could not parse PredictionString: {e}")
+
+
+def grade(submission: pd.DataFrame, answers: pd.DataFrame) -> float:
+    """
+    Grade the submission against the answers.
+
+    `submission` and `answers` are DataFrames with the following columns:
+    - `id` (str): The study ID
+    - `PredictionString` (str): Has the form "{label} {confidence} {xmin} {ymin} {xmax} {ymax}" for each bounding box
+      (so there may be 6 x N space-separated values in total, where N is the number of bounding boxes).
+    """
+    if len(submission) != len(answers):
+        raise InvalidSubmissionError("Submission and answers must have the same length")
+
+    if "id" not in submission.columns:
+        raise InvalidSubmissionError("Submission must have an 'Id' column")
+
+    if "PredictionString" not in submission.columns:
+        raise InvalidSubmissionError("Submission must have a 'PredictionString' column")
+
+    assert "id" in answers.columns, "Answers must have an 'id' column"
+    assert "PredictionString" in answers.columns, "Answers must have a 'PredictionString' column"
+
+    submission = submission.sort_values("id")
+    answers = answers.sort_values("id")
+    if (submission["id"].values != answers["id"].values).any():
+        raise InvalidSubmissionError("Submission and answers must have the same ids")
+
+    # Attempt to parse the PredictionString
+    submission_preds = submission["PredictionString"].apply(parse_prediction_string)
+    answers_preds = answers["PredictionString"].apply(parse_prediction_string)
+
+    # Calculate the mAP
+    return calculate_map(submission_preds, answers_preds) if len(submission_preds) > 0 else 0.0

dslighting/registry/siim-covid19-detection/prepare.py

@@ -0,0 +1,123 @@
+import shutil
+from pathlib import Path
+
+import pandas as pd
+from sklearn.model_selection import train_test_split
+from tqdm import tqdm
+
+from mlebench.utils import read_csv
+
+
+def prepare(raw: Path, public: Path, private: Path):
+    """
+    There are two tasks:
+    - Image level: Object detection problem - detect the presence of pneumonia in the image using bounding boxes
+    - Study level: Classification problem - classify the study into one of the four classes
+
+    Images in train/ and test/ are stored in paths with the form {study}/{series}/{image}.
+
+    Original train has 6,334 samples, and test "is of roughly the same scale as the training dataset".
+    We'll split the original train into a new train/test split with 90/10 ratio.
+
+    The split happens at the study level, with image level following accordingly.
+    """
+    DEV_MODE = False
+
+    # Create new train_study_level.csv
+    train_study = read_csv(raw / "train_study_level.csv")
+    if DEV_MODE:
+        # randomly sample 200 rows for development
+        train_study = train_study.sample(n=200, random_state=0)
+    new_train_study, new_test_study = train_test_split(train_study, test_size=0.1, random_state=0)
+    new_train_study = new_train_study.sort_values(by="id")
+    new_test_study = new_test_study.sort_values(by="id")
+    new_train_study.to_csv(public / "train_study_level.csv", index=False)
+
+    # Create new train_image_level.csv
+    train_image = read_csv(raw / "train_image_level.csv")
+    new_train_image = train_image[
+        (train_image["StudyInstanceUID"] + "_study").isin(new_train_study["id"])
+    ]
+    new_test_image = train_image[
+        (train_image["StudyInstanceUID"] + "_study").isin(new_test_study["id"])
+    ]
+    new_train_image = new_train_image.sort_values(by="id")
+    new_test_image = new_test_image.sort_values(by="id")
+    if not DEV_MODE:
+        assert len(new_train_image) + len(new_test_image) == len(
+            train_image
+        ), f"Expected {len(train_image)} images"
+    new_train_image.to_csv(public / "train_image_level.csv", index=False)
+
+    # Copy data with shutil
+    for study_id in tqdm(new_train_study["id"], desc="Copying train data"):
+        study_id = study_id.replace("_study", "")
+        shutil.copytree(raw / "train" / study_id, public / "train" / study_id)
+    for study_id in tqdm(new_test_study["id"], desc="Copying test data"):
+        study_id = study_id.replace("_study", "")
+        shutil.copytree(raw / "train" / study_id, public / "test" / study_id)
+    assert len(list(public.glob("train/*"))) == len(
+        new_train_study
+    ), f"Expected {len(new_train_study)} studies"
+    assert len(list(public.glob("test/*"))) == len(
+        new_test_study
+    ), f"Expected {len(new_test_study)} studies"
+
+    # Create gold answer submission
+    rows = []
+
+    """ 
+    # new_test_study currently looks like:
+    id,Negative for Pneumonia,Typical Appearance,Indeterminate Appearance,Atypical Appearance
+    00086460a852_study,0,1,0,0
+    000c9c05fd14_study,0,0,0,1
+    # but for the submission we need to convert it to the following, where label is one of "negative", "typical", "indeterminate", "atypical"
+    id,PredictionString
+    00188a671292_study,{label} 1 0 0 1 1
+    004bd59708be_study,{label} 1 0 0 1 1
+    """
+    for idx, row in new_test_study.iterrows():
+        label = ["negative", "typical", "indeterminate", "atypical"][row[1:].argmax()]
+        # Study-level task is just a classification task, so set bounding boxes all the same (1 0 0 1 1)
+        # then the metric will only care about the label
+        # https://www.kaggle.com/competitions/siim-covid19-detection/data
+        rows.append({"id": row["id"], "PredictionString": f"{label} 1 0 0 1 1"})
+
+    # new_test_image currently looks like this, and we only want the "label" column as the PredictionString
+    """
+    id,boxes,label,StudyInstanceUID
+    000a312787f2_image,"[{'x': 789.28836, 'y': 582.43035, 'width': 1026.65662, 'height': 1917.30292}, {'x': 2245.91208, 'y': 591.20528, 'width': 1094.66162, 'height': 1761.54944}]",opacity 1 789.28836 582.43035 1815.94498 2499.73327 opacity 1 2245.91208 591.20528 3340.5737 2352.75472,5776db0cec75
+    000c3a3f293f_image,,none 1 0 0 1 1,ff0879eb20ed
+    0012ff7358bc_image,"[{'x': 677.42216, 'y': 197.97662, 'width': 867.79767, 'height': 999.78214}, {'x': 1792.69064, 'y': 402.5525, 'width': 617.02734, 'height': 1204.358}]",opacity 1 677.42216 197.97662 1545.21983 1197.75876 opacity 1 1792.69064 402.5525 2409.71798 1606.9105,9d514ce429a7
+    """
+    for idx, row in new_test_image.iterrows():
+        rows.append({"id": row["id"], "PredictionString": row["label"]})
+
+    answers = pd.DataFrame(rows)
+    assert len(answers) == len(new_test_study) + len(
+        new_test_image
+    ), f"Expected {len(new_test_study) + len(new_test_image)} answers"
+    answers.to_csv(private / "test.csv", index=False)
+
+    # Create sample submission
+    """ Sample submission should look like:
+    id,PredictionString
+    00188a671292_study,negative 1 0 0 1 1
+    004bd59708be_study,negative 1 0 0 1 1
+    00508faccd39_study,negative 1 0 0 1 1
+    ...
+    f77d7d1aebab_image,none 1 0 0 1 1
+    ccc5b63ca96d_image,none 1 0 0 1 1
+    5e8ac1fe2b82_image,none 1 0 0 1 1
+    """
+    rows = []
+    for idx, row in new_test_study.iterrows():
+        rows.append({"id": row["id"], "PredictionString": "negative 1 0 0 1 1"})
+    for idx, row in new_test_image.iterrows():
+        rows.append({"id": row["id"], "PredictionString": "none 1 0 0 1 1"})
+
+    sample_submission = pd.DataFrame(rows)
+    assert len(sample_submission) == len(new_test_study) + len(
+        new_test_image
+    ), f"Expected {len(new_test_study) + len(new_test_image)} answers"
+    sample_submission.to_csv(public / "sample_submission.csv", index=False)

dslighting/registry/siim-covid19-detection/prepare_val.py

@@ -0,0 +1,164 @@
+import shutil
+from pathlib import Path
+
+import pandas as pd
+from sklearn.model_selection import train_test_split
+from tqdm import tqdm
+
+from mlebench.utils import read_csv
+
+
+def _process_split(
+    train_study_df: pd.DataFrame,
+    test_study_df: pd.DataFrame,
+    full_image_df: pd.DataFrame,
+    raw_data_root: Path,
+    output_public_path: Path,
+    output_private_path: Path,
+    dev_mode: bool,
+):
+    """
+    Helper function to process a single data split and generate all required output files.
+
+    Args:
+        train_study_df: DataFrame of studies for the training set.
+        test_study_df: DataFrame of studies for the test set.
+        full_image_df: DataFrame of all images, to be filtered for this split.
+        raw_data_root: Path to the original raw data.
+        output_public_path: Path to the public output directory.
+        output_private_path: Path to the private output directory.
+        dev_mode: Boolean flag for development mode.
+    """
+    # Create output directories if they don't exist
+    output_public_path.mkdir(parents=True, exist_ok=True)
+    (output_public_path / "train").mkdir(exist_ok=True)
+    (output_public_path / "test").mkdir(exist_ok=True)
+    output_private_path.mkdir(parents=True, exist_ok=True)
+
+    # Save the study-level train CSV
+    train_study_df.to_csv(output_public_path / "train_study_level.csv", index=False)
+
+    # Filter and save the image-level CSVs based on the study split
+    split_train_image = full_image_df[
+        (full_image_df["StudyInstanceUID"] + "_study").isin(train_study_df["id"])
+    ]
+    split_test_image = full_image_df[
+        (full_image_df["StudyInstanceUID"] + "_study").isin(test_study_df["id"])
+    ]
+    split_train_image = split_train_image.sort_values(by="id")
+    split_test_image = split_test_image.sort_values(by="id")
+    if not dev_mode:
+        assert len(split_train_image) + len(split_test_image) == len(
+            full_image_df[
+                (full_image_df["StudyInstanceUID"] + "_study").isin(
+                    pd.concat([train_study_df["id"], test_study_df["id"]])
+                )
+            ]
+        ), f"Image split mismatch"
+    split_train_image.to_csv(output_public_path / "train_image_level.csv", index=False)
+
+    # Copy image data for the split
+    for study_id in tqdm(train_study_df["id"], desc=f"Copying train data to {output_public_path.name}"):
+        study_id = study_id.replace("_study", "")
+        shutil.copytree(raw_data_root / "train" / study_id, output_public_path / "train" / study_id, dirs_exist_ok=True)
+    for study_id in tqdm(test_study_df["id"], desc=f"Copying test data to {output_public_path.name}"):
+        study_id = study_id.replace("_study", "")
+        shutil.copytree(raw_data_root / "train" / study_id, output_public_path / "test" / study_id, dirs_exist_ok=True)
+
+    assert len(list(output_public_path.glob("train/*"))) == len(
+        train_study_df
+    ), f"Expected {len(train_study_df)} studies in {output_public_path.name}/train"
+    assert len(list(output_public_path.glob("test/*"))) == len(
+        test_study_df
+    ), f"Expected {len(test_study_df)} studies in {output_public_path.name}/test"
+
+
+    # Create gold answer submission for the test set
+    rows = []
+    for idx, row in test_study_df.iterrows():
+        label = ["negative", "typical", "indeterminate", "atypical"][row[1:].argmax()]
+        rows.append({"id": row["id"], "PredictionString": f"{label} 1 0 0 1 1"})
+    for idx, row in split_test_image.iterrows():
+        rows.append({"id": row["id"], "PredictionString": row["label"]})
+
+    answers = pd.DataFrame(rows)
+    assert len(answers) == len(test_study_df) + len(
+        split_test_image
+    ), f"Expected {len(test_study_df) + len(split_test_image)} answers"
+    answers.to_csv(output_private_path / "test.csv", index=False)
+
+    # Create sample submission for the test set
+    rows = []
+    for idx, row in test_study_df.iterrows():
+        rows.append({"id": row["id"], "PredictionString": "negative 1 0 0 1 1"})
+    for idx, row in split_test_image.iterrows():
+        rows.append({"id": row["id"], "PredictionString": "none 1 0 0 1 1"})
+
+    sample_submission = pd.DataFrame(rows)
+    assert len(sample_submission) == len(test_study_df) + len(
+        split_test_image
+    ), f"Expected {len(test_study_df) + len(split_test_image)} sample answers"
+    sample_submission.to_csv(output_public_path / "sample_submission.csv", index=False)
+
+
+def prepare(raw: Path, public: Path, private: Path):
+    """
+    There are two tasks:
+    - Image level: Object detection problem - detect the presence of pneumonia in the image using bounding boxes
+    - Study level: Classification problem - classify the study into one of the four classes
+
+    Images in train/ and test/ are stored in paths with the form {study}/{series}/{image}.
+
+    Original train has 6,334 samples, and test "is of roughly the same scale as the training dataset".
+    We'll split the original train into a new train/test split with 90/10 ratio.
+
+    The split happens at the study level, with image level following accordingly.
+    """
+    DEV_MODE = False
+
+    # Read the original full datasets
+    train_study = read_csv(raw / "train_study_level.csv")
+    train_image = read_csv(raw / "train_image_level.csv")
+
+    if DEV_MODE:
+        # randomly sample 200 rows for development
+        train_study = train_study.sample(n=200, random_state=0)
+
+    # --- First Split: Create the main train/test sets ---
+    # This split creates the primary competition data, saved to `public` and `private`.
+    new_train_study, new_test_study = train_test_split(train_study, test_size=0.1, random_state=0)
+    new_train_study = new_train_study.sort_values(by="id").reset_index(drop=True)
+    new_test_study = new_test_study.sort_values(by="id").reset_index(drop=True)
+
+    _process_split(
+        train_study_df=new_train_study,
+        test_study_df=new_test_study,
+        full_image_df=train_image,
+        raw_data_root=raw,
+        output_public_path=public,
+        output_private_path=private,
+        dev_mode=DEV_MODE,
+    )
+
+    # --- Second Split: Create the validation train/test sets ---
+    # This split takes the *training set* from the first split (`new_train_study`)
+    # and splits it again to create a validation set. The outputs are saved to
+    # parallel `public_val` and `private_val` directories.
+    public_val = public.parent / "public_val"
+    private_val = private.parent / "private_val"
+
+    # We use the exact same splitting parameters to maintain consistency.
+    # The new test set (`test_val_study`) will be 10% of the `new_train_study`.
+    train_val_study, test_val_study = train_test_split(new_train_study, test_size=0.1, random_state=0)
+    train_val_study = train_val_study.sort_values(by="id").reset_index(drop=True)
+    test_val_study = test_val_study.sort_values(by="id").reset_index(drop=True)
+
+    _process_split(
+        train_study_df=train_val_study,
+        test_study_df=test_val_study,
+        full_image_df=train_image, # The full image set is used for filtering
+        raw_data_root=raw,
+        output_public_path=public_val,
+        output_private_path=private_val,
+        dev_mode=DEV_MODE,
+    )

dslighting/registry/siim-isic-melanoma-classification/checksums.yaml

@@ -0,0 +1,7 @@
+private:
+  test.csv: 89aae5dac5881e960258883de51152fe
+public:
+  sample_submission.csv: c8fa2aaa934d434e5b87e148ca91d42e
+  test.csv: faeb3cdaf1ee9c47120bcbf46787fe81
+  train.csv: dc473826f850a25d5e62f981b5e62d54
+zip: ed8a6c59a3677cada6365ecdb42d8d23

dslighting/registry/siim-isic-melanoma-classification/config.yaml

@@ -0,0 +1,26 @@
+id: siim-isic-melanoma-classification
+name: SIIM-ISIC Melanoma Classification
+competition_type: simple
+awards_medals: true
+prizes:
+  - position: 1
+    value: 10000
+  - position: 2
+    value: 6000
+  - position: 3
+    value: 4000
+  - position: With Context Special Prize
+    value: 5000
+  - position: Without Context Special Prize
+    value: 5000
+description: mlebench/competitions/siim-isic-melanoma-classification/description.md
+
+dataset:
+  answers: siim-isic-melanoma-classification/prepared/private/test.csv
+  sample_submission: siim-isic-melanoma-classification/prepared/public/sample_submission.csv
+
+grader:
+  name: auc-roc
+  grade_fn: mlebench.competitions.siim-isic-melanoma-classification.grade:grade
+
+preparer: mlebench.competitions.siim-isic-melanoma-classification.prepare:prepare