bbbc-datasets 0.1.0__py3-none-any.whl

bbbc_datasets/config.py

@@ -0,0 +1,15 @@
+BBBC_DATASETS = {
+    "BBBC001": {
+        "image_paths": [
+            "https://bbbc.broadinstitute.org/BBBC001/images_1",
+            "https://bbbc.broadinstitute.org/BBBC001/images_2",
+        ],
+        "label_path": "https://bbbc.broadinstitute.org/BBBC001/masks",
+        "local_path": "data/BBBC001",
+    },
+    "BBBC002": {
+        "image_paths": ["https://bbbc.broadinstitute.org/BBBC002/images"],
+        "label_path": None,  # No segmentation ground truth
+        "local_path": "data/BBBC002",
+    },
+}

bbbc_datasets/dataset_manager.py

@@ -0,0 +1,123 @@
+import cv2
+import numpy as np
+import tifffile as tiff
+import torch
+from torch.utils.data import Dataset
+
+from tests import DATASETS  # Import shared dataset list
+
+
+class BBBCDataset(Dataset):
+    """
+    PyTorch-compatible dataset for loading BBBC datasets with full 3D support.
+
+    - Supports both 2D and 3D images.
+    - Optionally applies transformations (PyTorch `torchvision.transforms`).
+    - Returns (image, label) pairs where available.
+
+    Args:
+        dataset_cls: The BBBC dataset class to load.
+        transform (callable, optional): Optional transform to apply to images.
+        target_transform (callable, optional): Optional transform for labels.
+    """
+
+    def __init__(self, dataset_cls, transform=None, target_transform=None):
+        self.dataset = dataset_cls()
+        self.image_paths = self.dataset.get_image_paths()
+        self.label_path = self.dataset.get_label_paths()
+
+        if not self.image_paths:
+            raise RuntimeError(f"No images found in {dataset_cls.__name__}")
+
+        self.transform = transform
+        self.target_transform = target_transform
+
+    def __len__(self):
+        return len(self.image_paths)
+
+    def __getitem__(self, idx):
+        image_path = self.image_paths[idx]
+        image = self.load_image(image_path)
+
+        label = None
+        if self.label_path:
+            label = self.load_image(self.label_path)
+
+        # Apply transforms if provided
+        if self.transform:
+            image = self.transform(image)
+        if self.target_transform and label is not None:
+            label = self.target_transform(label)
+
+        return image, label
+
+    def load_image(self, image_path):
+        """
+        Loads an image (2D or full 3D) and converts it to a PyTorch tensor.
+        """
+        if image_path.endswith((".tif", ".tiff")):
+            img = tiff.imread(image_path)  # Load 3D TIFF or 2D image
+        else:
+            img = cv2.imread(image_path, cv2.IMREAD_UNCHANGED)  # Load 2D image
+
+        if img is None:
+            raise ValueError(f"Error: Could not read image {image_path}")
+
+        # Normalize grayscale images
+        img = (img - np.min(img)) / (np.max(img) - np.min(img) + 1e-8)
+
+        # Convert to tensor
+        if len(img.shape) == 2:  # 2D grayscale image
+            img = torch.tensor(img, dtype=torch.float32).unsqueeze(0)  # (C, H, W)
+        elif len(img.shape) == 3:  # 3D volume (Z, H, W)
+            img = torch.tensor(img, dtype=torch.float32).unsqueeze(0)  # (C, Z, H, W)
+
+        return img
+
+
+class DatasetManager:
+    """
+    Manages all BBBC datasets and provides utilities.
+
+    - Lists available datasets.
+    - Loads datasets dynamically as PyTorch Datasets.
+    - Displays dataset information.
+    """
+
+    @staticmethod
+    def list_datasets():
+        """Prints a summary of available BBBC datasets."""
+        print("\n📂 Available BBBC Datasets:")
+        for dataset_cls in DATASETS:
+            dataset = dataset_cls()
+            print(f"- {dataset_cls.__name__}: {dataset.dataset_name}")
+
+    @staticmethod
+    def get_dataset(name, transform=None, target_transform=None):
+        """
+        Loads a dataset by name.
+
+        Args:
+            name (str): The dataset class name (e.g., "BBBC003").
+            transform: Optional image transformations.
+            target_transform: Optional label transformations.
+
+        Returns:
+            BBBCDataset instance.
+        """
+        for dataset_cls in DATASETS:
+            if dataset_cls.__name__ == name:
+                return BBBCDataset(dataset_cls, transform, target_transform)
+
+        raise ValueError(
+            f"Dataset {name} not found. Use DatasetManager.list_datasets() to see available datasets."
+        )
+
+
+if __name__ == "__main__":
+    # Example Usage
+    DatasetManager.list_datasets()
+
+    # Load a dataset
+    dataset = DatasetManager.get_dataset("BBBC003")
+    print(f"\nLoaded {len(dataset)} samples from BBBC003")

bbbc_datasets/datasets/base_dataset.py

@@ -0,0 +1,230 @@
+import os
+import zipfile
+import difflib
+
+import pandas as pd
+import numpy as np
+import requests
+from tqdm import tqdm
+
+from bbbc_datasets.utils.file_io import load_image
+
+
+class BaseBBBCDataset:
+    """
+    Base class for BBBC datasets, handling downloading, extraction, and file management.
+
+    - Automatically downloads dataset files if they are not available locally.
+    - Stores datasets in a shared system-wide folder (`/opt/bbbc_datasets/` or `~/.bbbc_datasets/`).
+    - Supports structured access to images, labels, and metadata.
+    """
+
+    # Define a shared system-wide storage location
+    GLOBAL_STORAGE_PATH = os.path.expanduser("~/.bbbc_datasets/")
+    IMAGE_SUBDIR = "images"
+    LABEL_SUBDIR = "labels"
+
+    IMAGE_FILTER = [".png", ".jpg", ".jpeg", ".tif", ".tiff", ".ics"]
+
+    local_path = None
+    label_path = None
+    image_paths = None
+    is_3d = False
+
+    def __init__(self, dataset_name):
+        """
+        Initialize the dataset with name and file paths.
+
+        :param dataset_name: The name of the dataset (e.g., "BBBC003").
+        :param dataset_info: Dictionary containing paths to images, label masks, and metadata.
+        """
+        self.ground_truth = None
+        self.dataset_name = dataset_name
+
+        if not self.local_path:
+            raise ValueError("local_path not defined")
+
+        # Ensure the dataset directory exists in the shared location
+        os.makedirs(self.local_path, exist_ok=True)
+
+        # Download missing files
+        self._download_files()
+
+        if self.label_path and isinstance(self.label_path, str):
+            local_file, unzip_folder = self.get_download_folder(
+                self.label_path, "label_path"
+            )
+            if self.label_path.endswith(".csv"):
+                self.ground_truth = local_file
+            elif self.label_path.endswith(".tif"):
+                self.ground_truth = local_file
+
+    def _download_files(self):
+        """
+        Checks for missing dataset files and downloads them if necessary.
+        """
+
+        if isinstance(self.image_paths, list):
+            for url in self.image_paths:
+                self._download_and_extract("image", url)
+        elif self.image_paths:
+            self._download_and_extract("image", self.image_paths)
+
+        if isinstance(self.label_path, list):
+            for url in self.label_path:
+                self._download_and_extract("label", url)
+        elif self.label_path:
+            self._download_and_extract("label", self.label_path)
+
+    def _download_and_extract(self, key, url):
+        """
+        Downloads and extracts a dataset file if it is missing.
+        """
+        if not url.startswith("http"):
+            return  # Skip invalid URLs
+
+        local_file, unzip_folder = self.get_download_folder(url, key)
+
+        if not os.path.exists(local_file):
+            print(f"Downloading {local_file}...")
+            response = requests.get(url, stream=True)
+            if response.status_code == 200:
+                total_size = int(response.headers.get("content-length", 0))
+                with open(local_file, "wb") as f, tqdm(
+                    desc=local_file,
+                    total=total_size,
+                    unit="B",
+                    unit_scale=True,
+                    unit_divisor=1024,
+                ) as bar:
+                    for chunk in response.iter_content(chunk_size=1024):
+                        if chunk:
+                            f.write(chunk)
+                            bar.update(len(chunk))
+            else:
+                raise FileNotFoundError(f"Failed to download {url}")
+
+        if os.path.exists(local_file) and not os.path.exists(unzip_folder):
+            # Extract if it's a zip file
+            if local_file.endswith(".zip"):
+                self._extract_zip(local_file, unzip_folder)
+
+    def get_download_folder(self, url, key):
+        local_file = os.path.join(self.local_path, os.path.basename(url))
+        folder_name = self.IMAGE_SUBDIR if "image" in key else self.LABEL_SUBDIR
+        unzip_folder = os.path.join(self.local_path, folder_name)
+        return local_file, unzip_folder
+
+    def _extract_zip(self, zip_path, extract_to=None):
+        """
+        Extracts a zip file to the specified directory or the dataset directory.
+        """
+        target_path = extract_to if extract_to else self.local_path
+        print(f"Extracting {zip_path} to {target_path}...")
+        with zipfile.ZipFile(zip_path, "r") as zip_ref:
+            zip_ref.extractall(target_path)
+
+    def _get_paths(self, subdir, recursive=True):
+        """
+        Returns the list of image file paths.
+        """
+
+        dir_path = os.path.join(self.local_path, subdir)
+        files = self._list_files(dir_path=dir_path)
+
+        # recursively search for images in the list of files
+        if recursive:
+            for file in files:
+                if os.path.isdir(file):
+                    files.extend(self._list_files(dir_path=file))
+
+        images = [f for f in files if f.lower().endswith(tuple(self.IMAGE_FILTER))]
+        return images
+
+    def get_image_paths(self):
+        """
+        Returns the list of image file paths.
+        """
+        images = self._get_paths(self.IMAGE_SUBDIR)
+        return images
+
+    def get_label(self, image_path):
+        """
+        Returns the label mask for a given image path.
+        """
+        if self.ground_truth:
+            if self.ground_truth.endswith(".tif"):
+                return load_image(self.ground_truth)
+            elif self.ground_truth.endswith(".csv"):
+                gt_all = pd.read_csv(self.ground_truth)
+                image_id = os.path.basename(image_path).split(".")[0]
+                gt_image = gt_all[gt_all["ImageId"] == image_id]
+                pixels = gt_image["EncodedPixels"]
+
+                image = load_image(image_path)
+                labels = np.zeros_like(image)
+
+                for pxls in pixels:
+                    if isinstance(pxls, str):
+                        pxls = pxls.split(" ")
+                        pxls = [int(p) for p in pxls]
+                        for i in range(0, len(pxls), 2):
+                            labels[pxls[i] : pxls[i + 1]] = 1
+                return labels
+            else:
+                raise NotImplementedError("Label type not supported.")
+
+        # Find the corresponding label mask
+        label_paths = self.get_label_paths()
+
+        if not label_paths:
+            return None
+
+        # find the label path the most similar to the image path
+        label_path = difflib.get_close_matches(
+            image_path, label_paths, n=1, cutoff=0.25
+        )
+
+        if not label_path:
+            raise FileNotFoundError(f"Label mask not found for {image_path}")
+        else:
+            label_path = label_path[0]
+
+        if os.path.exists(label_path):
+            return load_image(label_path)
+        else:
+            raise FileNotFoundError(f"Label mask not found for {image_path}")
+
+    def get_label_paths(self):
+        """
+        Returns the label mask file path (if available).
+        """
+        images = self._get_paths(self.LABEL_SUBDIR)
+        return images
+
+    def get_metadata_paths(self):
+        """
+        Returns the metadata file paths (if available).
+        """
+        dir_path = os.path.join(self.local_path, "metadata")
+        return self._list_files(dir_path=dir_path)
+
+    def _list_files(self, dir_path):
+        """
+        Returns a list of files in a specific dataset subdirectory.
+        """
+
+        if os.path.exists(dir_path):
+            return [os.path.join(dir_path, f) for f in os.listdir(dir_path)]
+        return []
+
+    @staticmethod
+    def validate_url(url):
+        """
+        Checks if a given URL is reachable.
+        """
+        try:
+            response = requests.head(url, allow_redirects=True, timeout=5)
+            return response.status_code == 200
+        except requests.RequestException:
+            return False

bbbc_datasets/datasets/bbbc003.py

@@ -0,0 +1,40 @@
+import os
+
+from bbbc_datasets.datasets.base_dataset import BaseBBBCDataset
+
+
+class BBBC003(BaseBBBCDataset):
+    """
+    BBBC003 Dataset: DIC microscopy images of mouse embryos for segmentation analysis.
+
+    - **Biological Application:**
+      Fluorescence microscopy cannot be used for embryo viability assessment in in vitro fertilization.
+      Instead, DIC imaging is used, though automatic segmentation remains a challenge due to shading effects.
+
+    - **Images:**
+      15 images of size 640x480 pixels, acquired using a Nikon Eclipse TE200 microscope
+      with a 20x, 0.45 NA objective lens.
+
+    - **Segmentations:**
+      Ground truth masks available for foreground segmentation.
+
+    - **Additional Data:**
+      Cell count data for each image is provided.
+
+    - **Source:** https://bbbc.broadinstitute.org/BBBC003
+    """
+
+    def __init__(self):
+        self.local_path = os.path.join(self.GLOBAL_STORAGE_PATH, "BBBC003")
+        self.image_paths = [
+            "https://data.broadinstitute.org/bbbc/BBBC003/BBBC003_v1_images.zip"
+        ]
+        self.label_path = (
+            "https://data.broadinstitute.org/bbbc/BBBC003/BBBC003_v1_foreground.zip"
+        )
+        self.metadata_paths = [
+            "https://data.broadinstitute.org/bbbc/BBBC003/BBBC003_v1_counts.txt"
+        ]
+        self.is_3d = False
+
+        super().__init__("BBBC003")

bbbc_datasets/datasets/bbbc004.py

@@ -0,0 +1,60 @@
+import os
+
+from bbbc_datasets.datasets.base_dataset import BaseBBBCDataset
+
+
+class BBBC004(BaseBBBCDataset):
+    """
+    BBBC004 Dataset: Synthetic fluorescence microscopy images of nuclei with varying clustering probabilities.
+
+    - **Biological Application:**
+      This dataset is used to evaluate the performance of nuclei segmentation and counting algorithms
+      under different levels of nuclear clustering.
+
+    - **Images:**
+      Five subsets of 20 images each, with overlap probabilities: [0, 0.15, 0.3, 0.45, 0.6].
+
+    - **Segmentations:**
+      Ground truth binary segmentation masks for foreground detection.
+
+    - **Ground Truth:**
+      Each image contains exactly 300 objects.
+
+    - **Source:** https://bbbc.broadinstitute.org/BBBC004
+    """
+
+    BASE_URL = "https://data.broadinstitute.org/bbbc/BBBC004"
+
+    OVERLAP_PROBABILITIES = {
+        0.00: "000",
+        0.15: "015",
+        0.30: "030",
+        0.45: "045",
+        0.60: "060",
+    }
+
+    def __init__(self, overlap_probability=0.00):
+        """
+        Initialize the dataset with a specific overlap probability.
+
+        :param overlap_probability: The clustering probability of nuclei (0, 0.15, 0.3, 0.45, 0.6)
+        """
+        if overlap_probability not in self.OVERLAP_PROBABILITIES:
+            raise ValueError(
+                f"Invalid overlap probability: {overlap_probability}. Choose from {list(self.OVERLAP_PROBABILITIES.keys())}"
+            )
+
+        self.prob_str = self.OVERLAP_PROBABILITIES[overlap_probability]
+        self.local_path = os.path.join(
+            self.GLOBAL_STORAGE_PATH, "BBBC004", f"prob_{self.prob_str}"
+        )
+        self.image_paths = [
+            os.path.join(self.BASE_URL, f"BBBC004_v1_{self.prob_str}_images.zip")
+        ]
+        self.label_path = os.path.join(
+            self.BASE_URL, f"BBBC004_v1_{self.prob_str}_foreground.zip"
+        )
+        self.metadata_paths = None
+        self.is_3d = False
+
+        super().__init__(f"BBBC004_{self.prob_str}")

bbbc_datasets/datasets/bbbc005.py

@@ -0,0 +1,40 @@
+import os
+
+from bbbc_datasets.datasets.base_dataset import BaseBBBCDataset
+
+
+class BBBC005(BaseBBBCDataset):
+    """
+    BBBC005 Dataset: Simulated fluorescence microscopy images with varying focus blur.
+
+    - **Biological Application:**
+      Evaluates the impact of focus blur on foreground/background segmentation in high-content screening.
+
+    - **Images:**
+      - 19,200 images with different levels of focus blur.
+      - Simulated using the SIMCEP platform.
+      - Plate layout nomenclature:
+        - **Row** = Blur level
+        - **Column** = Cell count.
+
+    - **Segmentations:**
+      - Ground truth masks available for fully in-focus images (**F1** files).
+      - Stored as binary masks (white = foreground, black = background).
+
+    - **Metadata:**
+      - Cell counts and blur levels are encoded in image filenames.
+      - Additional metadata is stored in `BBBC005_results_bray.csv`.
+
+    - **Source:** https://bbbc.broadinstitute.org/BBBC005
+    """
+
+    BASE_URL = "https://data.broadinstitute.org/bbbc/BBBC005"
+
+    def __init__(self):
+        self.local_path = os.path.join(self.GLOBAL_STORAGE_PATH, "BBBC005")
+        self.image_paths = [os.path.join(self.BASE_URL, "BBBC005_v1_images.zip")]
+        self.label_path = os.path.join(self.BASE_URL, "BBBC005_v1_ground_truth.zip")
+        self.metadata_paths = [os.path.join(self.BASE_URL, "BBBC005_results_bray.csv")]
+        self.is_3d = False
+
+        super().__init__("BBBC005")

bbbc_datasets/datasets/bbbc006.py

@@ -0,0 +1,54 @@
+import os
+
+from bbbc_datasets.datasets.base_dataset import BaseBBBCDataset
+
+
+class BBBC006(BaseBBBCDataset):
+    """
+    BBBC006 Dataset: Z-stack fluorescence microscopy images of human U2OS cells.
+
+    - **Biological Application:**
+      Evaluates the effect of focus variations on foreground segmentation in high-content screening.
+
+    - **Images:**
+      - 32 z-stack focal planes per field of view.
+      - Two fluorescence markers:
+        - Hoechst (w1): Stains nuclei.
+        - Phalloidin (w2): Stains cell bodies.
+
+    - **Segmentations:**
+      - Ground truth segmentation available for in-focus images (z = 11 to z = 23).
+      - Masks are 8-bit grayscale PNGs (each segmented nucleus has a unique integer label).
+
+    - **Metadata:**
+      - Nucleus count data and field of view information.
+
+    - **Source:** https://bbbc.broadinstitute.org/BBBC006
+    """
+
+    BASE_URL = "https://data.broadinstitute.org/bbbc/BBBC006"
+
+    def __init__(self, z_plane=16):
+        """
+        Initialize the dataset for a specific z-plane.
+
+        :param z_plane: The z-stack focal plane to download (0-32, default = 16 for optimal focus).
+        """
+        if not (0 <= z_plane <= 32):
+            raise ValueError(f"Invalid z-plane: {z_plane}. Choose between 0 and 32.")
+
+        self.z_plane = z_plane
+        self.local_path = os.path.join(
+            self.GLOBAL_STORAGE_PATH, "BBBC006", f"z_{z_plane:02}"
+        )
+        self.image_paths = [
+            os.path.join(self.BASE_URL, f"BBBC006_v1_images_z_{z_plane:02}.zip")
+        ]
+        self.label_path = os.path.join(self.BASE_URL, "BBBC006_v1_labels.zip")
+        self.metadata_paths = [
+            os.path.join(self.BASE_URL, "BBBC006_v1_counts.csv"),
+            os.path.join(self.BASE_URL, "BBBC006_results_bray.csv"),
+        ]
+        self.is_3d = False
+
+        super().__init__(f"BBBC006_z_{z_plane:02}")

bbbc_datasets/datasets/bbbc008.py

@@ -0,0 +1,35 @@
+import os
+
+from bbbc_datasets.datasets.base_dataset import BaseBBBCDataset
+
+
+class BBBC008(BaseBBBCDataset):
+    """
+    BBBC008 Dataset: Fluorescence microscopy images of human HT29 colon cancer cells.
+
+    - **Biological Application:**
+      This dataset is used for segmentation and cell counting of HT29 colon cancer cells.
+
+    - **Images:**
+      - 12 images, each containing three fluorescence channels:
+        - Hoechst (DNA, Channel 1).
+        - Phalloidin (Actin, Channel 3).
+        - pH3 (Mitotic cells, Channel 2 - Not used for segmentation).
+
+    - **Segmentations:**
+      - Foreground-background segmentation masks available.
+      - Manually refined after initial thresholding.
+
+    - **Source:** https://bbbc.broadinstitute.org/BBBC008
+    """
+
+    BASE_URL = "https://data.broadinstitute.org/bbbc/BBBC008"
+
+    def __init__(self):
+        self.local_path = os.path.join(self.GLOBAL_STORAGE_PATH, "BBBC008")
+        self.image_paths = [os.path.join(self.BASE_URL, "BBBC008_v1_images.zip")]
+        self.label_path = os.path.join(self.BASE_URL, "BBBC008_v1_foreground.zip")
+        self.metadata_paths = None
+        self.is_3d = False
+
+        super().__init__("BBBC008")

bbbc_datasets/datasets/bbbc010.py

@@ -0,0 +1,39 @@
+import os
+
+from bbbc_datasets.datasets.base_dataset import BaseBBBCDataset
+
+
+class BBBC010(BaseBBBCDataset):
+    """
+    BBBC010 Dataset: Live/dead assay of C. elegans exposed to pathogens.
+
+    - **Biological Application:**
+      Used for screening novel anti-infectives by distinguishing between live and dead C. elegans worms.
+      Positive control (ampicillin-treated worms) appear curved and smooth (alive).
+      Negative control (untreated worms) appear rod-like and textured (dead).
+
+    - **Images:**
+      - 100 images from a 384-well plate.
+      - Brightfield (Channel 1) and GFP (Channel 2).
+      - Image size: 696 x 520 pixels, 16-bit TIF format.
+
+    - **Segmentations:**
+      - **Foreground/background segmentation:** Human-corrected binary images.
+      - **Individual worm outlines:** Binary segmentation images for individual worms.
+
+    - **Source:** https://bbbc.broadinstitute.org/BBBC010
+    """
+
+    BASE_URL = "https://data.broadinstitute.org/bbbc/BBBC010"
+
+    def __init__(self):
+        self.local_path = os.path.join(self.GLOBAL_STORAGE_PATH, "BBBC010")
+        self.image_paths = [os.path.join(self.BASE_URL, "BBBC010_v2_images.zip")]
+        self.label_path = os.path.join(self.BASE_URL, "BBBC010_v1_foreground.zip")
+        self.additional_label_paths = [
+            os.path.join(self.BASE_URL, "BBBC010_v1_foreground_eachworm.zip")
+        ]
+        self.metadata_paths = None
+        self.is_3d = False
+
+        super().__init__("BBBC010")