plato-learn 1.1__py3-none-any.whl

plato/datasources/datalib/gym_utils/gym_trim.py

@@ -0,0 +1,189 @@
+"""
+
+Cut the whole video based on the requirements
+
+data_root = '../../../data/gym'
+video_root = f'{data_root}/videos'
+anno_root = f'{data_root}/annotations'
+anno_file = f'{anno_root}/annotation.json'
+
+event_anno_file = f'{anno_root}/event_annotation.json'
+event_root = f'{data_root}/events'
+
+"""
+
+import os
+import os.path as osp
+import subprocess
+
+import mmcv
+
+
+def trim_event(video_root, anno_file, event_anno_file, event_root):
+    """Trim the videos into many events"""
+    videos = os.listdir(video_root)
+    videos = set(videos)
+    annotation = mmcv.load(anno_file)
+    event_annotation = {}
+
+    mmcv.mkdir_or_exist(event_root)
+
+    for anno_key, anno_value in annotation.items():
+        if anno_key + ".mp4" not in videos:
+            print(f"video {anno_key} has not been downloaded")
+            continue
+
+        video_path = osp.join(video_root, anno_key + ".mp4")
+
+        for event_id, event_anno in anno_value.items():
+            timestamps = event_anno["timestamps"][0]
+            start_time, end_time = timestamps
+            event_name = anno_key + "_" + event_id
+
+            output_filename = event_name + ".mp4"
+
+            command = [
+                "ffmpeg",
+                "-i",
+                '"%s"' % video_path,
+                "-ss",
+                str(start_time),
+                "-t",
+                str(end_time - start_time),
+                "-c:v",
+                "libx264",
+                "-c:a",
+                "copy",
+                "-threads",
+                "8",
+                "-loglevel",
+                "panic",
+                '"%s"' % osp.join(event_root, output_filename),
+            ]
+            command = " ".join(command)
+            try:
+                subprocess.check_output(command, shell=True, stderr=subprocess.STDOUT)
+            except subprocess.CalledProcessError:
+                print(
+                    f"Trimming of the Event {event_name} of Video {anno_key} Failed",
+                    flush=True,
+                )
+
+            segments = event_anno["segments"]
+            if segments is not None:
+                event_annotation[event_name] = segments
+
+    mmcv.dump(event_annotation, event_anno_file)
+
+
+# data_root = '../../../data/gym'
+# anno_root = f'{data_root}/annotations'
+
+# event_anno_file = f'{anno_root}/event_annotation.json'
+# event_root = f'{data_root}/events'
+# subaction_root = f'{data_root}/subactions'
+
+
+def trim_subsection(event_anno_file, event_root, subaction_root):
+    """Further trim the event into several subsections"""
+    events = os.listdir(event_root)
+    events = set(events)
+    annotation = mmcv.load(event_anno_file)
+
+    mmcv.mkdir_or_exist(subaction_root)
+
+    for anno_key, anno_value in annotation.items():
+        if anno_key + ".mp4" not in events:
+            print(
+                f"video {anno_key[:11]} has not been downloaded "
+                f"or the event clip {anno_key} not generated"
+            )
+            continue
+
+        video_path = osp.join(event_root, anno_key + ".mp4")
+
+        for subaction_id, subaction_anno in anno_value.items():
+            timestamps = subaction_anno["timestamps"]
+            start_time, end_time = timestamps[0][0], timestamps[-1][1]
+            subaction_name = anno_key + "_" + subaction_id
+
+            output_filename = subaction_name + ".mp4"
+
+            command = [
+                "ffmpeg",
+                "-i",
+                '"%s"' % video_path,
+                "-ss",
+                str(start_time),
+                "-t",
+                str(end_time - start_time),
+                "-c:v",
+                "libx264",
+                "-c:a",
+                "copy",
+                "-threads",
+                "8",
+                "-loglevel",
+                "panic",
+                '"%s"' % osp.join(subaction_root, output_filename),
+            ]
+            command = " ".join(command)
+            try:
+                subprocess.check_output(command, shell=True, stderr=subprocess.STDOUT)
+            except subprocess.CalledProcessError:
+                print(
+                    f"Trimming of the Subaction {subaction_name} of Event "
+                    f"{anno_key} Failed",
+                    flush=True,
+                )
+
+
+def generate_splits_list(data_root, annotation_root, frame_data_root):
+    """Generate the split information based on the predefined files"""
+    videos = os.listdir(data_root)
+    videos = set(videos)
+    train_file_org = osp.join(annotation_root, "gym99_train_org.txt")
+    val_file_org = osp.join(annotation_root, "gym99_val_org.txt")
+    train_file = osp.join(annotation_root, "gym99_train.txt")
+    val_file = osp.join(annotation_root, "gym99_val.txt")
+    train_frame_file = osp.join(annotation_root, "gym99_train_rawframes.txt")
+    val_frame_file = osp.join(annotation_root, "gym99_val_rawframes.txt")
+
+    train_org = open(train_file_org).readlines()
+
+    train_org = [x.strip().split() for x in train_org]
+
+    train = [x for x in train_org if x[0] + ".mp4" in videos]
+
+    if osp.exists(frame_data_root):
+        train_frames = []
+        for line in train:
+            length = len(os.listdir(osp.join(frame_data_root, line[0])))
+            train_frames.append([line[0], str(length // 3), line[1]])
+        train_frames = [" ".join(x) for x in train_frames]
+        with open(train_frame_file, "w") as fout:
+            fout.write("\n".join(train_frames))
+
+    train = [x[0] + ".mp4 " + x[1] for x in train]
+
+    with open(train_file, "w") as fout:
+        fout.write("\n".join(train))
+
+    val_org = open(val_file_org).readlines()
+    val_org = [x.strip().split() for x in val_org]
+    val = [x for x in val_org if x[0] + ".mp4" in videos]
+
+    if osp.exists(frame_data_root):
+        val_frames = []
+        for line in val:
+            if not os.path.exists(osp.join(frame_data_root, line[0])):
+                continue
+            length = len(os.listdir(osp.join(frame_data_root, line[0])))
+            val_frames.append([line[0], str(length // 3), line[1]])
+        val_frames = [" ".join(x) for x in val_frames]
+        with open(val_frame_file, "w") as fout:
+            fout.write("\n".join(val_frames))
+
+    val = [x[0] + ".mp4 " + x[1] for x in val]
+    with open(val_file, "w") as fout:
+        fout.write("\n".join(val))

plato/datasources/datalib/modality_data_anntation_tools.py

@@ -0,0 +1,163 @@
+"""
+The class in this file is supported by the mmaction/tools/data/build_file_list
+
+
+"""
+
+import os
+import glob
+import json
+
+from mmaction.tools.data.anno_txt2json import lines2dictlist
+from mmaction.tools.data.parse_file_list import parse_directory
+
+from plato.datasources.datalib.parse_datasets import build_list, obtain_data_splits_info
+
+
+class GenerateMDataAnnotation(object):
+    """Generate the annotation file for the existing data modality"""
+
+    def __init__(
+        self,
+        data_src_dir,
+        data_annos_files_info,  # a dict that contains the data splits' file path
+        data_format,  # 'rawframes', 'videos'
+        out_path,
+        dataset_name,
+        data_dir_level=2,
+        rgb_prefix="img_'",  # prefix of rgb frames
+        flow_x_prefix="flow_x_",  # prefix of flow x frames [flow_x_ or x_]
+        flow_y_prefix="flow_y_",  # prefix of flow y frames [flow_y_ or y_]
+        # shuffle=False,  # whether to shuffle the file list
+        output_format="json",
+    ):  # txt or json
+        self.data_src_dir = data_src_dir
+        self.data_annos_files_info = data_annos_files_info
+        self.dataset_name = dataset_name
+        self.data_format = data_format
+        self.annotations_out_path = out_path
+        self.data_dir_level = data_dir_level
+        self.rgb_prefix = rgb_prefix
+        self.flow_x_prefix = flow_x_prefix
+        self.flow_y_prefix = flow_y_prefix
+
+        self.output_format = output_format
+
+        self.data_splits_info = None
+        self.frame_info = None
+
+    def read_data_splits_csv_info(self):
+        """Get the data splits information from the csv annotation files"""
+        self.data_splits_info = obtain_data_splits_info(
+            data_annos_files_info=self.data_annos_files_info,
+            data_fir_level=2,
+            data_name=self.dataset_name,
+        )
+
+    def parse_levels_dir(self, data_src_dir):
+        data_dir_info = {}
+        """ Parse the dir with several levels. """
+        if self.data_dir_level == 1:
+            # search for one-level directory
+            files_list = glob.glob(os.path.join(data_src_dir, "*"))
+        elif self.data_dir_level == 2:
+            # search for two-level directory
+            files_list = glob.glob(os.path.join(data_src_dir, "*", "*"))
+        else:
+            raise ValueError(f"level must be 1 or 2, but got {self.data_dir_level}")
+        for file in files_list:
+            file_path = os.path.relpath(file, data_src_dir)
+            # for video: video_id: (video_relative_path, -1, -1)
+            # for audio: audio_id: (audio_relative_path, -1, -1)
+            data_dir_info[os.path.splitext(file_path)[0]] = (file_path, -1, -1)
+
+        return data_dir_info
+
+    def parse_dir_files(self, split):
+        """Parse the dir to summary the data information"""
+        # The annotations for audio spectrogram features are identical to those of rawframes.
+
+        # data_format = "rawframes" if self.data_format == "audio_features" else self.data_format
+        # split_format_data_src_dir = os.path.join(self.data_src_dir, split,
+        #                                          data_format)
+
+        split_format_data_src_dir = os.path.join(
+            self.data_src_dir, split, self.data_format
+        )
+        frame_info = None
+        if self.data_format == "rawframes":
+            frame_info = parse_directory(
+                split_format_data_src_dir,
+                rgb_prefix=self.rgb_prefix,
+                flow_x_prefix=self.flow_x_prefix,
+                flow_y_prefix=self.flow_y_prefix,
+                level=self.data_dir_level,
+            )
+        elif self.data_format == "videos":
+            frame_info = self.parse_levels_dir(split_format_data_src_dir)
+        elif self.data_format in ["audio_features", "audios"]:
+            # the audio anno list should be consistent with that of rawframes
+            rawframes_src_path = os.path.join(self.data_src_dir, split, "rawframes")
+            frame_info = parse_directory(
+                rawframes_src_path,
+                rgb_prefix=self.rgb_prefix,
+                flow_x_prefix=self.flow_x_prefix,
+                flow_y_prefix=self.flow_y_prefix,
+                level=self.data_dir_level,
+            )
+        else:
+            raise NotImplementedError("only rawframes and videos are supported")
+        self.frame_info = frame_info
+
+    def get_anno_file_path(self, split_name):
+        """Get the annotation file path"""
+        filename = f"{self.dataset_name}_{split_name}_list_{self.data_format}.txt"
+
+        if self.output_format == "json":
+            filename = filename.replace(".txt", ".json")
+
+        output_anno_file_path = os.path.join(self.annotations_out_path, filename)
+
+        return output_anno_file_path
+
+    def generate_data_splits_info_file(self, split_name):
+        """Generate the data split information and write the info to file"""
+        self.parse_dir_files(split_name)
+
+        split_info = self.data_splits_info[split_name]
+
+        # (rgb_list, flow_list)
+        split_built_list = build_list(
+            split=split_info, frame_info=self.frame_info, shuffle=False
+        )
+
+        output_file_path = self.get_anno_file_path(split_name=split_name)
+
+        data_format = (
+            "rawframes"
+            if self.data_format in ["audio_features", "audios"]
+            else self.data_format
+        )
+
+        if self.output_format == "txt":
+            with open(output_file_path, "w") as anno_file:
+                anno_file.writelines(split_built_list[0])
+        elif self.output_format == "json":
+            data_list = lines2dictlist(split_built_list[0], data_format)
+            if self.data_format in ["audios", "audio_features"]:
+
+                def change_title_func(elem):
+                    """Using this function to"""
+                    # added the filename key with value presenting the
+                    #  path of the corresponding video
+                    if self.data_format == "audio_features":
+                        elem["audio_path"] = elem["frame_dir"] + ".npy"
+                    else:
+                        elem["audio_path"] = elem["frame_dir"] + ".wav"
+
+                    return elem
+
+                data_list = [change_title_func(elem) for elem in data_list]
+
+            with open(output_file_path, "w") as anno_file:
+                json.dump(data_list, anno_file)

plato/datasources/datalib/modality_extraction_base.py

@@ -0,0 +1,59 @@
+"""
+Classes for parsing the structured files for the data
+
+"""
+
+import glob
+import logging
+import os
+
+
+class VideoExtractorBase:
+    """The base class for the following video extractor classes"""
+
+    def __init__(
+        self, video_src_dir, dir_level=2, num_worker=8, video_ext="mp4", mixed_ext=False
+    ):
+        self.video_src_dir = video_src_dir
+        self.dir_level = dir_level
+        self.num_worker = num_worker
+        self.video_ext = video_ext  # support 'avi', 'mp4', 'webm'
+        self.mixed_ext = mixed_ext
+
+        # assert self.dir_level == 2  # we insist two-level data directory setting
+
+        logging.info("Reading videos from folder: %s", self.video_src_dir)
+        if self.mixed_ext:
+            logging.info("Using the mixture extensions of videos")
+            fullpath_list = glob.glob(self.video_src_dir + "/*" * self.dir_level)
+        else:
+            logging.info("Using the mixture extensions of videos: %s", self.video_ext)
+            fullpath_list = glob.glob(
+                self.video_src_dir + "/*" * self.dir_level + "." + self.video_ext
+            )
+
+        logging.info("Total number of videos found: %s", len(fullpath_list))
+
+        # the full path list is the full path of the video,
+        # for example: ./data/Kinetics/Kinetics700/train/video/clay_pottery_making/RE6YNPccYK4.mp4',
+        self.fullpath_list = fullpath_list
+
+        # Video item containing video full path,
+        # for example: clay_pottery_making/RE6YNPccYK4.mp4
+        self.videos_path_list = list(
+            map(
+                lambda p: os.path.join(
+                    os.path.basename(os.path.dirname(p)), os.path.basename(p)
+                ),
+                self.fullpath_list,
+            )
+        )
+
+    def organize_modality_dir(self, src_dir, to_dir):
+        """Organize the data dir of the modality into two level - calssname/data_id"""
+
+        classes = os.listdir(src_dir)
+        for classname in classes:
+            new_dir = os.path.join(to_dir, classname)
+            if not os.path.isdir(new_dir):
+                os.makedirs(new_dir)

plato/datasources/datalib/parse_datasets.py

@@ -0,0 +1,212 @@
+"""This part of the code heavily depends on the
+tools/data/build_file_lists.py provided by the mmaction
+
+"""
+
+import csv
+import random
+
+from mmaction.tools.data.parse_file_list import (
+    parse_diving48_splits,
+    parse_hmdb51_split,
+    parse_jester_splits,
+    parse_mit_splits,
+    parse_mmit_splits,
+    parse_sthv1_splits,
+    parse_sthv2_splits,
+    parse_ucf101_splits,
+)
+
+
+def build_list(split, frame_info, shuffle=False):
+    """Build RGB and Flow file list with a given split.
+
+    Args:
+        split (list): Split to be generate file list.
+
+    Returns:
+        tuple[list, list]: (rgb_list, flow_list), rgb_list is the
+            generated file list for rgb, flow_list is the generated
+            file list for flow.
+    """
+    rgb_list, flow_list = list(), list()
+    for item in split:
+        if item[0] not in frame_info:
+            continue
+        elif frame_info[item[0]][1] > 0:
+            # rawframes
+            rgb_cnt = frame_info[item[0]][1]
+            flow_cnt = frame_info[item[0]][2]
+            if isinstance(item[1], int):
+                rgb_list.append(f"{item[0]} {rgb_cnt} {item[1]}\n")
+                flow_list.append(f"{item[0]} {flow_cnt} {item[1]}\n")
+            elif isinstance(item[1], list):
+                # only for multi-label datasets like mmit
+                rgb_list.append(
+                    f"{item[0]} {rgb_cnt} "
+                    + " ".join([str(digit) for digit in item[1]])
+                    + "\n"
+                )
+                rgb_list.append(
+                    f"{item[0]} {flow_cnt} "
+                    + " ".join([str(digit) for digit in item[1]])
+                    + "\n"
+                )
+            else:
+                raise ValueError(
+                    "frame_info should be "
+                    + "[`video`(str), `label`(int)|`labels(list[int])`"
+                )
+        else:
+            # videos
+            if isinstance(item[1], int):
+                rgb_list.append(f"{frame_info[item[0]][0]} {item[1]}\n")
+                flow_list.append(f"{frame_info[item[0]][0]} {item[1]}\n")
+            elif isinstance(item[1], list):
+                # only for multi-label datasets like mmit
+                rgb_list.append(
+                    f"{frame_info[item[0]][0]} "
+                    + " ".join([str(digit) for digit in item[1]])
+                    + "\n"
+                )
+                flow_list.append(
+                    f"{frame_info[item[0]][0]} "
+                    + " ".join([str(digit) for digit in item[1]])
+                    + "\n"
+                )
+            else:
+                raise ValueError(
+                    "frame_info should be "
+                    + "[`video`(str), `label`(int)|`labels(list[int])`"
+                )
+    if shuffle:
+        random.shuffle(rgb_list)
+        random.shuffle(flow_list)
+    return (rgb_list, flow_list)
+
+
+def parse_kinetics_splits(kinetics_anntation_files_info, level, dataset_name):
+    """Parse Kinetics dataset into "train", "val", "test" splits.
+
+    Args:
+        kinetics_anntation_files_info (dict): The file path of the original annotation file.
+                                        The file should be the "*.csv" provided in the
+                                        official website.
+                                        For example:
+                                            {"train": ""}
+        level (int): Directory level of data. 1 for the single-level directory,
+            2 for the two-level directory.
+        dataset (str): Denotes the version of Kinetics that needs to be parsed,
+            choices are "kinetics400", "kinetics600" and "kinetics700".
+
+    Returns:
+        list: "train", "val", "test" splits of Kinetics.
+    """
+
+    def convert_label(label_str, keep_whitespaces=False):
+        """Convert label name to a formal string.
+
+        Remove redundant '"' and convert whitespace to '_'.
+
+        Args:
+            label_str (str): String to be converted.
+            keep_whitespaces(bool): Whether to keep whitespace. Default: False.
+
+        Returns:
+            str: Converted string.
+        """
+        if not keep_whitespaces:
+            return label_str.replace('"', "").replace(" ", "_")
+        else:
+            return label_str.replace('"', "")
+
+    def line_to_map(line_str, test=False):
+        """A function to map line string to video and label.
+
+        Args:
+            line_str (str): A single line from Kinetics csv file.
+            test (bool): Indicate whether the line comes from test
+                annotation file.
+
+        Returns:
+            tuple[str, str]: (video, label), video is the video id,
+                label is the video label.
+        """
+        if test:  # x:  ['---v8pgm1eQ', '0', '10', 'test']
+            # video = f'{x[0]}_{int(x[1]):06d}_{int(x[2]):06d}'
+            # video = f'{x[1]}_{int(float(x[2])):06d}_{int(float(x[3])):06d}'
+            video = f"{line_str[0]}_{int(float(line_str[1])):06d}_{int(float(line_str[2])):06d}"
+            label = -1  # label unknown
+            return video, label
+        else:  # ['clay pottery making', '---0dWlqevI', '19', '29', 'train']
+            video = f"{line_str[1]}_{int(float(line_str[2])):06d}_{int(float(line_str[3])):06d}"
+            if level == 2:
+                video = f"{convert_label(line_str[0])}/{video}"
+            else:
+                assert level == 1
+            label = class_mapping[convert_label(line_str[0])]
+            return video, label
+
+    train_file = kinetics_anntation_files_info["train"]
+    test_file = kinetics_anntation_files_info["test"]
+    val_file = kinetics_anntation_files_info["val"]
+
+    csv_reader = csv.reader(open(train_file))
+    # skip the first line
+    next(csv_reader)
+
+    labels_sorted = sorted({convert_label(row[0]) for row in csv_reader})
+    class_mapping = {label: i for i, label in enumerate(labels_sorted)}
+
+    csv_reader = csv.reader(open(train_file))
+    next(csv_reader)
+    train_list = [line_to_map(x) for x in csv_reader]
+
+    csv_reader = csv.reader(open(val_file))
+    next(csv_reader)
+    val_list = [line_to_map(x) for x in csv_reader]
+
+    csv_reader = csv.reader(open(test_file))
+    next(csv_reader)
+    test_list = [line_to_map(x, test=True) for x in csv_reader]
+
+    splits = ((train_list, val_list, test_list),)
+    splits = {"train": train_list, "test": test_list, "val": val_list}
+    return splits
+
+
+def obtain_data_splits_info(
+    data_annos_files_info,  # a dict containing the data original splits' file path
+    data_fir_level=2,
+    data_name="kinetics700",
+):
+    """Parse the raw data file to obtain different splits info"""
+    if data_name == "ucf101":
+        splits = parse_ucf101_splits(data_fir_level)
+    elif data_name == "sthv1":
+        splits = parse_sthv1_splits(data_fir_level)
+    elif data_name == "sthv2":
+        splits = parse_sthv2_splits(data_fir_level)
+    elif data_name == "mit":
+        splits = parse_mit_splits()
+    elif data_name == "mmit":
+        splits = parse_mmit_splits()
+    elif data_name in ["kinetics400", "kinetics600", "kinetics700"]:
+        kinetics_anntation_files_info = data_annos_files_info
+        splits = parse_kinetics_splits(
+            kinetics_anntation_files_info, data_fir_level, data_name
+        )
+    elif data_name == "hmdb51":
+        splits = parse_hmdb51_split(data_fir_level)
+    elif data_name == "jester":
+        splits = parse_jester_splits(data_fir_level)
+    elif data_name == "diving48":
+        splits = parse_diving48_splits()
+    else:
+        raise ValueError(
+            f"Supported datasets are 'ucf101, sthv1, sthv2', 'jester', "
+            f"'mmit', 'mit', 'kinetics400', 'kinetics600', 'kinetics700', but "
+            f"got {data_name}"
+        )
+
+    return splits