plato-learn 1.1__py3-none-any.whl

plato/datasources/cifar100.py

@@ -0,0 +1,61 @@
+"""
+The CIFAR-100 dataset from the torchvision package.
+"""
+
+from torchvision import datasets, transforms
+
+from plato.config import Config
+from plato.datasources import base
+
+
+class DataSource(base.DataSource):
+    """The CIFAR-100 dataset."""
+
+    def __init__(self, **kwargs):
+        super().__init__()
+        _path = Config().params["data_path"]
+
+        train_transform = (
+            kwargs["train_transform"]
+            if "train_transform" in kwargs
+            else (
+                transforms.Compose(
+                    [
+                        transforms.RandomHorizontalFlip(),
+                        transforms.RandomCrop(32, 4),
+                        transforms.ToTensor(),
+                        transforms.Normalize(
+                            [0.485, 0.456, 0.406], [0.229, 0.224, 0.225]
+                        ),
+                    ]
+                )
+            )
+        )
+
+        test_transform = (
+            kwargs["test_transform"]
+            if "test_transform" in kwargs
+            else (
+                transforms.Compose(
+                    [
+                        transforms.ToTensor(),
+                        transforms.Normalize(
+                            [0.485, 0.456, 0.406], [0.229, 0.224, 0.225]
+                        ),
+                    ]
+                )
+            )
+        )
+
+        self.trainset = datasets.CIFAR100(
+            root=_path, train=True, download=True, transform=train_transform
+        )
+        self.testset = datasets.CIFAR100(
+            root=_path, train=False, download=True, transform=test_transform
+        )
+
+    def num_train_examples(self):
+        return 50000
+
+    def num_test_examples(self):
+        return 10000

plato/datasources/cinic10.py

@@ -0,0 +1,62 @@
+"""
+The CINIC-10 dataset.
+
+For more information about CINIC-10, refer to:
+
+https://github.com/BayesWatch/cinic-10
+"""
+
+import logging
+import os
+
+from torchvision import datasets, transforms
+
+from plato.config import Config
+from plato.datasources import base
+
+
+class DataSource(base.DataSource):
+    """The CINIC-10 dataset."""
+
+    def __init__(self, **kwargs):
+        super().__init__()
+        _path = Config().params["data_path"]
+
+        if not os.path.exists(_path):
+            logging.info("Downloading the CINIC-10 dataset. This may take a while.")
+            url = (
+                Config().data.download_url
+                if hasattr(Config().data, "download_url")
+                else "http://iqua.ece.toronto.edu/baochun/CINIC-10.tar.gz"
+            )
+            DataSource.download(url, _path)
+
+        train_transform = (
+            kwargs["train_transform"]
+            if "train_transform" in kwargs
+            else (
+                transforms.Compose(
+                    [
+                        transforms.ToTensor(),
+                        transforms.Normalize(
+                            [0.47889522, 0.47227842, 0.43047404],
+                            [0.24205776, 0.23828046, 0.25874835],
+                        ),
+                    ]
+                )
+            )
+        )
+        test_transform = train_transform
+
+        self.trainset = datasets.ImageFolder(
+            root=os.path.join(_path, "train"), transform=train_transform
+        )
+        self.testset = datasets.ImageFolder(
+            root=os.path.join(_path, "test"), transform=test_transform
+        )
+
+    def num_train_examples(self):
+        return 90000
+
+    def num_test_examples(self):
+        return 90000

plato/datasources/coco.py

@@ -0,0 +1,119 @@
+"""
+The MS COCO- dataset stands for Common Objects in Context, and is
+  designed to represent a vast array of objects that we
+  regularly encounter in everyday life.
+
+We mainly utilize COCO-17 (25.20 GB):
+ - COCO has 121,408 images in total.
+ - has 883,331 object annotations
+ - COCO defines 91 classes but the data only uses 80 classes.
+ - Some images from the train and validation sets don’t have annotations.
+ - The test set does not have annotations.
+ - COCO 2014 and 2017 use the same images, but the splits are different.
+ -> for image, detection, segmentation.
+
+The data structure and setting follows:
+ "https://cocodataset.org/#home".
+
+Then, the download urls are obtained from:
+ "https://gist.github.com/mkocabas/a6177fc00315403d31572e17700d7fd9".
+
+We utilize the official splits that contain:
+ - train: 118,287 images
+ - val: 5,000 images
+ - test: 40,670 images
+
+The file structure of this dataset is:
+ - train2017: train images
+ - test2017: test images
+ - val2017: validation images
+ - annotations_trainval2017: captions for train/val
+
+The data structure under the 'data/' is:
+├── COCO2017           # root dir of COCO2017 Entities dataset
+│   ├── COCO2017Raw    # Raw images/annotations and the official splits
+│   │   └── annotations
+│   │   └── train2017
+│   │   └── test2017
+│   │   └── val2017
+│   ├── train       # images for the train phase
+│   └── test        # images for the test phase
+│   └── val         # images for the validation phase
+
+Note:
+    Currently, we have not utilize the COCO dataset to train the model.
+     Thus, we only implement the code of downloading and arrange the data,
+     which is required when using the referitgame dataset.
+"""
+
+import os
+import shutil
+
+from plato.config import Config
+from plato.datasources import multimodal_base
+
+
+class DataSource(multimodal_base.MultiModalDataSource):
+    """The COCO dataset."""
+
+    def __init__(self, **kwargs):
+        super().__init__()
+
+        self.data_name = Config().data.dataname
+        self.data_source = Config().data.datasource
+
+        self.modality_names = ["image", "text"]
+
+        _path = Config().params["data_path"]
+        self._data_path_process(data_path=_path, base_data_name=self.data_name)
+
+        base_data_path = self.mm_data_info["data_path"]
+        raw_data_name = self.data_name + "Raw"
+        raw_data_path = os.path.join(base_data_path, raw_data_name)
+        if not self._exists(raw_data_path):
+            os.makedirs(raw_data_path, exist_ok=True)
+
+        download_train_url = Config().data.download_train_url
+        download_test_url = Config().data.download_test_url
+        download_val_url = Config().data.download_val_url
+        download_annotation_url = Config().data.download_annotation_url
+
+        splits_downalods = {
+            "train": download_train_url,
+            "test": download_test_url,
+            "val": download_val_url,
+        }
+
+        # Download raw data and extract to different splits
+        for split_name in list(self.splits_info.keys()):
+            split_path = self.splits_info[split_name]["path"]
+            split_download_url = splits_downalods[split_name]
+            split_file_name = self._download_arrange_data(
+                download_url_address=split_download_url,
+                data_path=raw_data_path,
+                extract_to_dir=split_path,
+            )
+            # renaming the extracted file to "images"
+            extracted_path = os.path.join(split_path, split_file_name)
+            renamed_path = os.path.join(split_path, "images")
+            os.rename(src=extracted_path, dst=renamed_path)
+
+        # Download the annotation
+        self._download_arrange_data(
+            download_url_address=download_annotation_url, data_path=raw_data_path
+        )
+        annotation_path = os.path.join(raw_data_path, "annotations")
+
+        # Move the annotation to each split
+        splits_caption_name = {
+            "train": "captions_train2017.json",
+            "val": "captions_val2017.json",
+        }
+        for split_name in list(splits_caption_name.keys()):
+            split_caption_name = splits_caption_name[split_name]
+            to_split_path = os.path.join(
+                self.splits_info[split_name]["path"], "captions.json"
+            )
+            shutil.copyfile(
+                src=os.path.join(annotation_path, split_caption_name), dst=to_split_path
+            )

plato/datasources/datalib/audio_extraction_tools.py

@@ -0,0 +1,137 @@
+"""
+Tools for extracting information from the audio
+
+"""
+
+import glob
+import os
+
+from multiprocessing import Pool
+
+from mmaction.tools.data import build_audio_features
+
+from plato.datasources.datalib import modality_extraction_base
+
+
+def obtain_audio_dest_dir(out_dir, audio_path, dir_level):
+    """Get the destination path to save the audio"""
+
+    if dir_level == 2:
+        class_name = os.path.basename(os.path.dirname(audio_path))
+        _, tail = os.path.split(audio_path)
+        _ = tail.split(".")[0]  # should get the audio_name
+        out_dir_full_path = os.path.join(out_dir, class_name)
+    else:  # the class name is not contained
+        _ = audio_path.split(".")[0]  # audio_name
+        out_dir_full_path = out_dir
+
+    return out_dir_full_path
+
+
+def extract_audio_wav(line_times):
+    """Extract the audio wave from video streams using FFMPEG."""
+    line, root, out_dir = line_times
+    video_id, _ = os.path.splitext(os.path.basename(line))
+    video_dir = os.path.dirname(line)
+    video_rel_dir = os.path.relpath(video_dir, root)
+    dst_dir = os.path.join(out_dir, video_rel_dir)
+    os.popen(f"mkdir -p {dst_dir}")
+    try:
+        if os.path.exists(f"{dst_dir}/{video_id}.wav"):
+            return
+        cmd = f"ffmpeg -i {line}  -map 0:a  -y {dst_dir}/{video_id}.wav"
+        os.popen(cmd)
+    except OSError:
+        with open("extract_wav_err_file.txt", "a+") as error_file:
+            error_file.write(f"{line}\n")
+
+
+class VideoAudioExtractor(modality_extraction_base.VideoExtractorBase):
+    """A class for extracting audio from the video"""
+
+    def __init__(
+        self,
+        video_src_dir,
+        dir_level=2,
+        num_worker=2,
+        video_ext="mp4",
+        mixed_ext=False,
+        audio_ext="wav",
+    ):
+        super().__init__(video_src_dir, dir_level, num_worker, video_ext, mixed_ext)
+        self.audio_ext = audio_ext
+
+    def build_audios(
+        self,
+        to_dir,
+    ):
+        """Extract audios in parallel"""
+        sourc_video_dir = self.video_src_dir
+        if self.dir_level == 2:
+            self.organize_modality_dir(src_dir=sourc_video_dir, to_dir=to_dir)
+        _ = glob.glob(to_dir + "/*" * self.dir_level + ".wav")
+
+        pool = Pool(self.num_worker)
+        pool.map(
+            extract_audio_wav,
+            zip(
+                self.fullpath_list,
+                len(self.videos_path_list) * [sourc_video_dir],
+                len(self.videos_path_list) * [to_dir],
+            ),
+        )
+
+    def build_audios_features(
+        self,
+        audio_src_path,  # the dir that contains the src audio files
+        to_dir,  # dir to save the extracted features
+        frame_rate=30,  # The frame rate per second of the video.
+        sample_rate=16000,  # The sample rate for audio sampling
+        num_mels=80,  # Number of channels of the melspectrogram. Default
+        fft_size=1280,  # fft_size / sample_rate is window size
+        hop_size=320,  # hop_size / sample_rate is step size
+        spectrogram_type="lws",  # lws, 'librosa', recommand lws
+        # part: Determines how many parts to be splited and which part to run.
+        # e.g., 2/5 means splitting all files into 5-fold and executing the 2nd part.
+        # This is useful if you have several machines
+        part="1/1",
+    ):
+        """Obtain the feature from the audio"""
+        audio_tools = build_audio_features.AudioTools(
+            frame_rate=frame_rate,
+            sample_rate=sample_rate,
+            num_mels=num_mels,
+            fft_size=fft_size,
+            hop_size=hop_size,
+            spectrogram_type=spectrogram_type,
+        )
+
+        audio_files = glob.glob(
+            audio_src_path + "/*" * self.dir_level + "." + self.audio_ext
+        )
+        files = sorted(audio_files)
+
+        if part is not None:
+            [this_part, num_parts] = [int(i) for i in part.split("/")]
+            part_len = len(files) // num_parts
+
+        extractor_pool = Pool(self.num_worker)
+        for file in files[
+            part_len * (this_part - 1) : (part_len * this_part)
+            if this_part != num_parts
+            else len(files)
+        ]:
+            out_full_path = obtain_audio_dest_dir(
+                out_dir=to_dir, audio_path=file, dir_level=self.dir_level
+            )
+
+            # create the output dir if not existed
+            if not os.path.exists(out_full_path):
+                os.makedirs(out_full_path)
+
+            extractor_pool.apply_async(
+                build_audio_features.extract_audio_feature,
+                args=(file, audio_tools, out_full_path),
+            )
+        extractor_pool.close()
+        extractor_pool.join()

plato/datasources/datalib/data_utils.py

@@ -0,0 +1,124 @@
+"""
+Useful tools for processing the data
+
+"""
+
+import shutil
+import os
+import json
+import numpy as np
+
+
+def config_to_dict(plato_config):
+    """Convert the plato config (can be nested one) instance to the dict."""
+    # convert the whole to dict - OrderedDict
+    plato_config_dict = plato_config._asdict()
+
+    def to_dict(elem):
+        for key, value in elem.items():
+            try:
+                value = value._asdict()
+                elem[key] = to_dict(value)
+            except:
+                pass
+            if isinstance(value, list):
+                for idx, value_item in enumerate(value):
+                    try:
+                        value_item = value_item._asdict()
+                        value[idx] = to_dict(value_item)
+                    except:
+                        pass
+                elem[key] = value
+        return elem
+
+    plato_config_dict = to_dict(plato_config_dict)
+
+    return plato_config_dict
+
+
+def dict_list2tuple(dict_obj):
+    """Convert all list element in the dict to tuple"""
+    for key, value in dict_obj.items():
+        if isinstance(value, dict):
+            for inner_key, inner_v in value.items():
+                if isinstance(inner_v, list):
+                    # empty or None list, mainly for meta_keys
+                    if not value or inner_v[0] is None:
+                        dict_obj[key][inner_key] = ()
+                    else:
+                        dict_obj[key][inner_key] = tuple(inner_v)
+        else:
+            if isinstance(value, list):
+                # empty or None list, mainly for meta_keys
+                if not value or value[0] is None:
+                    dict_obj[key] = ()
+                else:
+                    dict_obj[key] = tuple(value)
+                for idx, item in enumerate(value):
+                    item = value[idx]
+                    if isinstance(item, dict):
+                        value[idx] = dict_list2tuple(item)
+
+    return dict_obj
+
+
+def phrase_boxes_alignment(flatten_boxes, ori_phrases_boxes):
+    """Align the phase and its corresponding boxes"""
+    phrases_boxes = list()
+
+    ori_pb_boxes_count = list()
+    for ph_boxes in ori_phrases_boxes:
+        ori_pb_boxes_count.append(len(ph_boxes))
+
+    strat_point = 0
+    for pb_boxes_num in ori_pb_boxes_count:
+        sub_boxes = list()
+        for i in range(strat_point, strat_point + pb_boxes_num):
+            sub_boxes.append(flatten_boxes[i])
+
+        strat_point += pb_boxes_num
+        phrases_boxes.append(sub_boxes)
+
+    pb_boxes_count = list()
+    for ph_boxes in phrases_boxes:
+        pb_boxes_count.append(len(ph_boxes))
+
+    assert pb_boxes_count == ori_pb_boxes_count
+
+    return phrases_boxes
+
+
+def list_inorder(listed_files, flag_str):
+    """ " List the files in order based on the file name"""
+    filtered_listed_files = [fn for fn in listed_files if flag_str in fn]
+    listed_files = sorted(filtered_listed_files, key=lambda x: x.strip().split(".")[0])
+    return listed_files
+
+
+def copy_files(src_files, dst_dir):
+    """copy files from src to dst"""
+    for file in src_files:
+        shutil.copy(file, dst_dir)
+
+
+def union_shuffled_lists(src_lists):
+    """shuffle the lists"""
+    for i in range(1, len(src_lists)):
+        assert len(src_lists[i]) == len(src_lists[i - 1])
+    processed = np.random.permutation(len(src_lists[0]))
+
+    return [np.array(ele)[processed] for ele in src_lists]
+
+
+def read_anno_file(anno_file_path):
+    _, tail = os.path.split(anno_file_path)
+    file_type = tail.split(".")[-1]
+
+    if file_type == "json":
+        with open(anno_file_path, "r") as anno_file:
+            annos_list = json.load(anno_file)
+    else:
+        with open(anno_file_path, "r") as anno_file:
+            annos_list = anno_file.readlines()
+
+    return annos_list