plato-learn 1.1__py3-none-any.whl

plato/datasources/multimodal_base.py

@@ -0,0 +1,328 @@
+"""
+Base class for multimodal datasets.
+"""
+
+from abc import abstractmethod
+import logging
+import os
+import subprocess
+from collections import namedtuple
+
+import torch
+from torchvision.datasets.utils import download_url, extract_archive
+from torchvision.datasets.utils import download_file_from_google_drive
+
+from plato.datasources import base
+
+TextData = namedtuple("TextData", ["caption", "caption_phrases"])
+BoxData = namedtuple("BoxData", ["caption_phrase_bboxs"])
+TargetData = namedtuple(
+    "TargetData", ["caption_phrases_cate", "caption_phrases_cate_id"]
+)
+
+
+class MultiModalDataSource(base.DataSource):
+    """
+    The training or testing dataset that accommodates custom augmentation and transforms.
+    """
+
+    def __init__(self):
+        super().__init__()
+
+        # data name
+        self.data_name = ""
+
+        # the text name of the contained modalities
+        self.modality_names = []
+
+        # define the information container for the source data
+        #  - source_data_path: the original downloaded data
+        #  - data_path: the source data used for the model
+        # For some datasets, we directly utilize the data_path as
+        #  there is no need to process the original downloaded data to put them
+        #  in the data_path dir.
+        self.mm_data_info = {"source_data_path": "", "data_path": ""}
+
+        # define the paths for the splited root data - train, test, and val
+        self.splits_info = {
+            "train": {"path": "", "split_anno_file": ""},
+            "test": {"path": "", "split_anno_file": ""},
+            "val": {"path": "", "split_anno_file": ""},
+        }
+
+    def set_modality_format(self, modality_name):
+        """An interface to set the modality name
+        Thus, calling this func to obtain the modality name
+         in all parts of the class to achieve the consistency
+        """
+        if modality_name in ["rgb", "flow"]:
+            modality_format = "rawframes"
+        else:  # convert to plurality
+            modality_format = modality_name + "s"
+
+        return modality_format
+
+    def set_modality_path_key_format(self, modality_name):
+        """An interface to set the modality path
+        Thus, calling this func to obtain the modality path
+         in all parts of the class to achieve the consistency
+        """
+        modality_format = self.set_modality_format(modality_name)
+
+        return modality_format + "_" + "path"
+
+    def _create_modalities_path(self, modality_names=None):
+        if modality_names is None:
+            assert len(self.modality_names) != 0
+            modality_names = self.modality_names
+
+        for split_type in list(self.splits_info.keys()):
+            split_path = self.splits_info[split_type]["path"]
+            for modality_nm in modality_names:
+                modality_format = self.set_modality_format(modality_nm)
+                split_modality_path = os.path.join(split_path, modality_format)
+                # modality data dir
+                modality_path_format = self.set_modality_path_key_format(modality_nm)
+                self.splits_info[split_type][modality_path_format] = split_modality_path
+                if not os.path.exists(split_modality_path):
+                    try:
+                        os.makedirs(split_modality_path)
+                    except FileExistsError:
+                        pass
+
+    def _data_path_process(
+        self,
+        data_path,
+        base_data_name=None,  # the base directory for the data
+    ):  # the directory name of the working data
+        """Generate the data structure based on the defined data path"""
+
+        # Create the full path by introducing the project path
+        base_data_path = os.path.join(data_path, base_data_name)
+
+        if not os.path.exists(base_data_path):
+            os.makedirs(base_data_path)
+
+        #
+        self.mm_data_info["data_path"] = base_data_path
+
+        # create the split dirs for current dataset
+        for split_type in list(self.splits_info.keys()):
+            split_path = os.path.join(base_data_path, split_type)
+            self.splits_info[split_type]["path"] = split_path
+            if not os.path.exists(split_path):
+                try:
+                    os.makedirs(split_path)
+                except FileExistsError:
+                    pass
+
+    def _download_arrange_data(
+        self,
+        download_url_address,
+        data_path,
+        extract_to_dir=None,
+        obtained_file_name=None,
+    ):
+        """Download the raw data and arrange the data"""
+        # Extract to the same dir as the download dir
+        if extract_to_dir is None:
+            extract_to_dir = data_path
+
+        download_file_name = os.path.basename(download_url_address)
+        download_file_path = os.path.join(data_path, download_file_name)
+
+        download_extracted_file_name = download_file_name.split(".")[0]
+        download_extracted_path = os.path.join(
+            extract_to_dir, download_extracted_file_name
+        )
+        # Download the raw data if necessary
+        if not self._exists(download_file_path):
+            logging.info("Downloading the %s data.....", download_file_name)
+            download_url(
+                url=download_url_address, root=data_path, filename=obtained_file_name
+            )
+
+        # Extract the data to the specific dir
+        if ".zip" in download_file_name or ".tar.gz" in download_file_name:
+            if not self._exists(download_extracted_path):
+                logging.info("Extracting data to %s dir.....", extract_to_dir)
+                extract_archive(
+                    from_path=download_file_path,
+                    to_path=extract_to_dir,
+                    remove_finished=False,
+                )
+
+        return download_extracted_file_name
+
+    def _download_google_driver_arrange_data(
+        self,
+        download_file_id,
+        extract_download_file_name,
+        data_path,
+    ):
+        download_data_file_name = extract_download_file_name + ".zip"
+        download_data_path = os.path.join(data_path, download_data_file_name)
+        extract_data_path = os.path.join(data_path, extract_download_file_name)
+        if not self._exists(download_data_path):
+            logging.info("Downloading the data to %s", download_data_path)
+            download_file_from_google_drive(
+                file_id=download_file_id,
+                root=data_path,
+                filename=download_data_file_name,
+            )
+        if not self._exists(extract_data_path):
+            extract_archive(
+                from_path=download_data_path, to_path=data_path, remove_finished=True
+            )
+
+    def _file_exists(self, tg_file_name, search_path, is_partial_name=True):
+        """Judge whether the input file exists in the search_path."""
+        # the tg_file_name matches one file if it match part of the file name
+        if is_partial_name:
+            is_included = lambda src_f_name: tg_file_name in src_f_name
+        else:
+            is_included = lambda src_f_name: tg_file_name == src_f_name
+        exists = any(is_included(f_name) for f_name in os.listdir(search_path))
+
+        return exists
+
+    def _exists(self, target_path):
+        """Does the input path/file exist and does the file contain useful data?"""
+        if not os.path.exists(target_path):
+            logging.info("The path %s does not exist.", target_path)
+            return False
+
+        # remove all .DS_Store files
+        command = ["find", ".", "-name", '".DS_Store"', "-delete"]
+        command = " ".join(command)
+        # cmd = f"find . -name ".DS_Store" -delete"
+        subprocess.check_output(command, shell=True, stderr=subprocess.STDOUT)
+
+        def get_size(folder):
+            # get size
+            size = 0
+            for ele in os.scandir(folder):
+                if not ele.name.startswith("."):
+                    size += os.path.getsize(ele)
+            return size
+
+        def is_contain_useful_file(target_dir):
+            """Return True once reaching one useful file"""
+            for __, __, files in os.walk(target_dir):
+                for file in files:
+                    # whether a useful file
+                    if not file.startswith("."):
+                        return True
+            return False
+
+        if os.path.isdir(target_path):
+            if get_size(target_path) == 0 or not is_contain_useful_file(target_path):
+                logging.info("The path %s exists but contains no data.", target_path)
+                return False
+
+            return True
+
+        logging.info("The file %s exists.", target_path)
+        return True
+
+    def num_modalities(self) -> int:
+        """The number of modalities."""
+        return len(self.modality_names)
+
+    @abstractmethod
+    def get_phase_dataset(self, phase, modality_sampler):
+        """Obtain the dataset with the modaltiy_sampler for the
+        specific phase (train/test/val)"""
+        raise NotImplementedError("Please implement the 'get_phase_dataset' method.")
+
+    @abstractmethod
+    def get_train_set(self, modality_sampler):
+        """Obtain the train dataset with the modaltiy_sampler"""
+        raise NotImplementedError("Please implement the 'get_train_set' method.")
+
+    @abstractmethod
+    def get_test_set(self, modality_sampler):
+        """Obtain the test dataset with the modaltiy_sampler."""
+        raise NotImplementedError("Please implement the 'get_test_set' method.")
+
+
+class MultiModalDataset(torch.utils.data.Dataset):
+    """The base interface for multimodal data."""
+
+    def __init__(self):
+        self.phase = None  # the 'train' , 'test', 'val'
+
+        # The recorded samples for current dataset:
+        #   In flickr20K entities dataset, this presents as:
+        #    this is a dict in which key is the 'sample name/id' ...
+        #    the values are the sample's information,
+        #    for example: the annotation with its bounding boxes ...
+        #   In the Kinetics, this presents as:
+        #    this is a dict:
+        #    {"rgb": rgb_dataset, "flow": flow_dataset, "audio": audio_dataset}
+        self.phase_multimodal_data_record = None
+
+        # Detailed information in selected split:
+        #   i.e., path, path for different modalities, etc.
+        self.phase_info = None
+        # the data types included,
+        #  e.g. in flickr30k entities, ["Images", "Annotations", "Sentences"]
+        self.data_types = None
+
+        # the name of the modalities in the dataset
+        self.modalities_name = None
+
+        # the sampler for modalities,
+        #   specific modalities can be masked by this sampler
+        self.modality_sampler = None
+        # transformation func for image and text if provided
+        self.transform_image_dec_func = None
+        self.transform_text_func = None
+
+        # the basic modalities
+        self.basic_modalities = ["rgb", "flow", "text", "audio"]
+        # the additional data/annotations
+        self.basic_items = ["box", "target"]
+
+    @abstractmethod
+    def get_targets(self):
+        """Obtain the labels of samples in current phase dataset."""
+        raise NotImplementedError("Please Implement the 'targets' function")
+
+    @abstractmethod
+    def get_one_multimodal_sample(self, sample_idx):
+        """Get the sample containing different modalities.
+        Different multi-modal datasets should have their
+         personal 'get_one_multimodal_sample' method.
+
+
+        Args:
+            sample_idx (int): the index of the sample
+
+        Output:
+            a dict containing different modalities, the
+             key of the dict is the modality name that should
+             be included in the basic_modalities and basic_items.
+        """
+        raise NotImplementedError(
+            "Please Implement the 'get_one_multimodal_sample(self, sample_idx)' function"
+        )
+
+    def __getitem__(self, sample_idx):
+        """Get the sample for either training or testing given index."""
+        sampled_multimodal_data = self.get_one_multimodal_sample(sample_idx)
+
+        # utilize the modality to mask specific modalities
+        sampled_modality_data = {}
+        for item_name, item_data in sampled_multimodal_data.items():
+            # maintain the modality data based on the sampler
+            # maintain the external data
+            if item_name in self.modality_sampler or item_name in self.basic_items:
+                sampled_modality_data[item_name] = item_data
+
+        return sampled_modality_data
+
+    @abstractmethod
+    def __len__(self):
+        """obtain the length of the multi-modal data"""
+        raise NotImplementedError("Please Implement this method")

plato/datasources/pascal_voc.py

@@ -0,0 +1,56 @@
+"""
+The PASCAL VOC dataset for image segmentation.
+"""
+
+from torchvision import datasets, transforms
+from plato.config import Config
+
+from plato.datasources import base
+
+
+class DataSource(base.DataSource):
+    """The PASCAL dataset."""
+
+    def __init__(self, **kwargs):
+        super().__init__()
+        _path = Config().params["data_path"]
+        self.mean = [0.45734706, 0.43338275, 0.40058118]
+        self.std = [0.23965294, 0.23532275, 0.2398498]
+
+        train_transform = (
+            kwargs["train_transform"]
+            if train_transform in kwargs
+            else (
+                transforms.Compose(
+                    [
+                        transforms.Resize((96, 96)),
+                        transforms.ToTensor(),
+                    ]
+                )
+            )
+        )
+
+        test_transform = train_transform
+
+        self.trainset = datasets.VOCSegmentation(
+            root=_path,
+            year="2012",
+            image_set="train",
+            download=True,
+            transform=train_transform,
+            target_transform=train_transform,
+        )
+        self.testset = datasets.VOCSegmentation(
+            root=_path,
+            year="2012",
+            image_set="val",
+            download=True,
+            transform=test_transform,
+            target_transform=test_transform,
+        )
+
+    def num_train_examples(self):
+        return len(self.trainset)
+
+    def num_test_examples(self):
+        return len(self.testset)

plato/datasources/purchase.py

@@ -0,0 +1,94 @@
+"""
+The Purchase100 dataset.
+"""
+
+import os
+import logging
+import urllib
+import tarfile
+import torch
+import numpy as np
+from torch.utils import data
+from plato.config import Config
+from plato.datasources import base
+
+
+class DataSource(base.DataSource):
+    """The Purchase100 dataset."""
+
+    def __init__(self, **kwargs):
+        super().__init__()
+        root_path = Config().params["data_path"]
+        dataset_path = os.path.join(root_path, "dataset_purchase")
+        if not os.path.isdir(root_path):
+            os.mkdir(root_path)
+        if not os.path.isfile(dataset_path):
+            self.download_dataset(root_path, dataset_path)
+
+        self.trainset, self.testset = self.extract_data(root_path)
+
+    def download_dataset(self, root_path, dataset_path):
+        """Download the Purchase100 dataset."""
+        logging.info("Downloading the Purchase100 dataset...")
+        filename = "https://www.comp.nus.edu.sg/~reza/files/dataset_purchase.tgz"
+        urllib.request.urlretrieve(
+            filename, os.path.join(root_path, "tmp_purchase.tgz")
+        )
+        logging.info("Dataset downloaded.")
+        tar = tarfile.open(os.path.join(root_path, "tmp_purchase.tgz"))
+        tar.extractall(path=root_path)
+
+        logging.info("Processing the dataset...")
+        data_set = np.genfromtxt(dataset_path, delimiter=",")
+        logging.info("Finish processing the dataset.")
+
+        X = data_set[:, 1:].astype(np.float64)
+        Y = (data_set[:, 0]).astype(np.int32) - 1
+        np.savez(os.path.join(root_path, "purchase_numpy.npz"), X=X, Y=Y)
+
+    def extract_data(self, root_path):
+        """Extract data."""
+        dataset = np.load(os.path.join(root_path, "purchase_numpy.npz"))
+
+        ## randomly shuffle the data
+        X, Y = dataset["X"], dataset["Y"]
+        np.random.seed(0)
+        indices = np.arange(len(X))
+        np.random.shuffle(indices)
+        X, Y = X[indices], Y[indices]
+
+        ## extract 20000 data samplers for training and testing respectively
+        num_train = 20000
+        train_data = X[:num_train]
+        test_data = X[num_train : num_train * 2]
+        train_label = Y[:num_train]
+        test_label = Y[num_train : num_train * 2]
+
+        ## create datasets
+        train_dataset = VectorDataset(train_data, train_label)
+        test_dataset = VectorDataset(test_data, test_label)
+
+        return train_dataset, test_dataset
+
+    def num_train_examples(self):
+        return 20000
+
+    def num_test_examples(self):
+        return 20000
+
+
+class VectorDataset(data.Dataset):
+    """
+    Create a Purchase100 dataset based on features and labels
+    """
+
+    def __init__(self, features, labels):
+        self.data = torch.stack([torch.FloatTensor(i) for i in features])
+        self.targets = torch.stack([torch.LongTensor([i]) for i in labels])[:, 0]
+        self.classes = [f"Style #{i}" for i in range(100)]
+
+    def __getitem__(self, index):
+        return self.data[index], self.targets[index]
+
+    def __len__(self):
+        return self.data.size(0)

plato/datasources/qoenflx.py

@@ -0,0 +1,127 @@
+"""
+The LIVE Netflix Video QoE datasets.
+
+For more information about the datasets, refer to
+https://live.ece.utexas.edu/research/LIVE_NFLXStudy/nflx_index.html.
+"""
+
+import copy
+import logging
+import os
+import re
+
+import numpy as np
+import scipy.io as sio
+import torch
+
+from plato.config import Config
+from plato.datasources import base
+
+FEATURE_NAMES = ["VQA", "R$_1$", "R$_2$", "M", "I"]
+
+
+class QoENFLXDataset(torch.utils.data.Dataset):
+    def __init__(self, dataset):
+        self.dataset = dataset
+
+    def __len__(self):
+        return len(self.dataset)
+
+    def __getitem__(self, idx):
+        VQA = torch.from_numpy(self.dataset[idx, [0]].astype(np.float)).float()
+        R1 = torch.from_numpy(self.dataset[idx, [1]].astype(np.float)).float()
+        R2 = self.dataset[idx, [2]].astype(np.int)
+        M = torch.from_numpy(self.dataset[idx, [3]].astype(np.float)).float()
+        I = torch.from_numpy(self.dataset[idx, [4]].astype(np.float)).float()
+        label = self.dataset[idx, [5]]
+        sample = {"VQA": VQA, "R1": R1, "R2": R2, "Mem": M, "Impair": I, "label": label}
+
+        return sample
+
+
+class DataSource(base.DataSource):
+    """A data source for QoENFLX datasets."""
+
+    def __init__(self, **kwargs):
+        super().__init__()
+
+        logging.info("Dataset: QoENFLX")
+        dataset_path = Config().params["data_path"] + "/QoENFLX/VideoATLAS/"
+        db_files = os.listdir(dataset_path)
+        db_files.sort(
+            key=lambda var: [
+                int(x) if x.isdigit() else x for x in re.findall(r"[^0-9]|[0-9]+", var)
+            ]
+        )
+        Nvideos = len(db_files)
+
+        pre_load_train_test_data_LIVE_Netflix = sio.loadmat(
+            Config().params["data_path"]
+            + "/QoENFLX/TrainingMatrix_LIVENetflix_1000_trials.mat"
+        )["TrainingMatrix_LIVENetflix_1000_trials"]
+
+        # randomly pick a trial out of the 1000
+        nt_rand = np.random.choice(
+            np.shape(pre_load_train_test_data_LIVE_Netflix)[1], 1
+        )
+        n_train = [
+            ind
+            for ind in range(0, Nvideos)
+            if pre_load_train_test_data_LIVE_Netflix[ind, nt_rand] == 1
+        ]
+        n_test = [
+            ind
+            for ind in range(0, Nvideos)
+            if pre_load_train_test_data_LIVE_Netflix[ind, nt_rand] == 0
+        ]
+
+        X = np.zeros((len(db_files), len(FEATURE_NAMES)))
+        y = np.zeros((len(db_files), 1))
+
+        feature_labels = list()
+        for typ in FEATURE_NAMES:
+            if typ == "VQA":
+                feature_labels.append("STRRED" + "_" + "mean")
+            elif typ == "R$_1$":
+                feature_labels.append("ds_norm")
+            elif typ == "R$_2$":
+                feature_labels.append("ns")
+            elif typ == "M":
+                feature_labels.append("tsl_norm")
+            else:
+                feature_labels.append("lt_norm")
+
+        for i, f in enumerate(db_files):
+            data = sio.loadmat(dataset_path + f)
+            for feat_cnt, feat in enumerate(feature_labels):
+                X[i, feat_cnt] = data[feat]
+            y[i] = data["final_subj_score"]
+
+        X_train_before_scaling = X[n_train, :]
+        X_test_before_scaling = X[n_test, :]
+        y_train = y[n_train]
+        y_test = y[n_test]
+
+        self.trainset = copy.deepcopy(
+            np.concatenate((X_train_before_scaling, y_train), axis=1)
+        )
+        self.testset = copy.deepcopy(
+            np.concatenate((X_test_before_scaling, y_test), axis=1)
+        )
+
+    @staticmethod
+    def get_train_loader(batch_size, trainset, sampler, shuffle=False):
+        """The custom train loader for QoENFLX."""
+        return torch.utils.data.DataLoader(
+            QoENFLXDataset(trainset),
+            batch_size=batch_size,
+            sampler=sampler,
+            shuffle=shuffle,
+        )
+
+    @staticmethod
+    def get_test_loader(batch_size, testset):
+        """The custom test loader for QoENFLX."""
+        return torch.utils.data.DataLoader(
+            QoENFLXDataset(testset), batch_size=batch_size, shuffle=False
+        )