modelmakerlibrary 0.1.0__py3-none-any.whl

src/dataset_function.py

@@ -0,0 +1,614 @@
+from src.packages import torch, tsv, csv, json, pd, os, datasets, v2, DataLoader
+
+class _FolderDataset(torch.utils.data.Dataset):
+    def __init__(self, path, model_type):
+        extensions = {
+            "text": {".txt"},
+            "video": {".mp4", ".mkv", ".avi", ".mov", ".webm", ".mpeg", ".mpg"},
+            "audio": {".wav", ".mp3", ".flac", ".ogg", ".m4a", ".aac"},
+        }[model_type]
+        self.path = path
+        self.model_type = model_type
+        self.classes = sorted(
+            name for name in os.listdir(path)
+            if os.path.isdir(os.path.join(path, name))
+        )
+        if not self.classes:
+            self.classes = [""]
+        self.class_to_idx = {name: index for index, name in enumerate(self.classes)}
+        self.samples = []
+        for class_name in self.classes:
+            class_path = os.path.join(path, class_name) if class_name else path
+            for root, _, files in os.walk(class_path):
+                for filename in files:
+                    if os.path.splitext(filename)[1].lower() in extensions:
+                        self.samples.append((os.path.join(root, filename), self.class_to_idx[class_name]))
+        self.samples.sort()
+
+    def __len__(self):
+        return len(self.samples)
+
+    def __getitem__(self, index):
+        path, label = self.samples[index]
+        if self.model_type == "text":
+            with open(path, encoding="utf-8") as file:
+                value = file.read()
+        else:
+            value = path
+        return value, torch.tensor(label)
+
+
+class _TextDataset(torch.utils.data.Dataset):
+    def __init__(self, dataset_path):
+        supported_extensions = {".txt", ".json", ".csv", ".tsv"}
+        if os.path.isfile(dataset_path):
+            paths = [dataset_path]
+            root = os.path.dirname(dataset_path)
+        else:
+            root = dataset_path
+            paths = sorted(
+                os.path.join(directory, filename)
+                for directory, _, filenames in os.walk(root)
+                for filename in filenames
+                if os.path.splitext(filename)[1].lower() in supported_extensions
+            )
+
+        self.samples = []
+        self.classes = []
+        class_indices = {}
+        empty_files = []
+        for path in paths:
+            extension = os.path.splitext(path)[1].lower()
+            relative = os.path.relpath(path, root)
+            parent = os.path.dirname(relative)
+            label = parent.split(os.sep, 1)[0] if parent else ""
+
+            if os.path.getsize(path) == 0:
+                empty_files.append(path)
+                continue
+            if extension == ".txt":
+                with open(path, encoding="utf-8") as file:
+                    text = file.read()
+                records = [(text, label)] if text.strip() else []
+            elif extension == ".json":
+                with open(path, encoding="utf-8") as file:
+                    data = json.load(file)
+                if isinstance(data, dict):
+                    data = data.get("data", [data])
+                if not isinstance(data, list):
+                    raise ValueError(f"Expected a JSON object or list of text records in '{path}'.")
+                records = [
+                    (item["text"], item.get("label", label))
+                    for item in data
+                    if isinstance(item, dict) and isinstance(item.get("text"), str) and item["text"].strip()
+                ]
+            else:
+                with open(path, encoding="utf-8", newline="") as file:
+                    reader = csv.DictReader(file, delimiter="\t" if extension == ".tsv" else ",")
+                    if not reader.fieldnames or "text" not in reader.fieldnames:
+                        raise ValueError(f"Expected a 'text' column in '{path}'.")
+                    records = [
+                        (row["text"], row.get("label", label))
+                        for row in reader
+                        if row.get("text") and row["text"].strip()
+                    ]
+
+            for text, record_label in records:
+                if record_label not in class_indices:
+                    class_indices[record_label] = len(self.classes)
+                    self.classes.append(record_label)
+                self.samples.append((text, class_indices[record_label]))
+        if not self.samples:
+            if empty_files:
+                details = ", ".join(f"'{path}'" for path in empty_files[:5])
+                raise ValueError(f"No non-empty text records found. Empty text file(s): {details}. Add text to the file(s) or use a dataset file containing text records.")
+            raise ValueError(f"No supported text files found under '{os.path.abspath(dataset_path)}'.")
+
+    def __len__(self):
+        return len(self.samples)
+
+    def __getitem__(self, index):
+        text, label = self.samples[index]
+        return text, torch.tensor(label)
+
+
+# Initialize the dataset by checking if the dataset path exists and replacing the dataset name accordingly
+def init_dataset(dataset_name, dataset_path):
+    dataset_path = os.fspath(dataset_path)
+    print("Initialized dataset with name '{}' and path '{}'.".format(dataset_name, os.path.abspath(dataset_path)))
+    return dataset_name, dataset_path
+
+
+
+def load_dataset(dataset_path, model_type="text"):
+    """Load a file or folder dataset for use with split_dataset or data_loader."""
+    if dataset_path is None or not os.path.exists(dataset_path):
+        raise ValueError(f"Dataset path does not exist: {dataset_path!r}.")
+    if model_type == "image":
+        if not os.path.isdir(dataset_path):
+            raise ValueError("Image datasets must be directories organized in class folders.")
+        dataset = datasets.ImageFolder(dataset_path, transform=v2.ToTensor())
+    elif model_type == "text":
+        dataset = _TextDataset(os.fspath(dataset_path))
+    elif model_type in {"video", "audio"}:
+        if not os.path.isdir(dataset_path):
+            raise ValueError(f"{model_type.capitalize()} datasets must be directories.")
+        dataset = _FolderDataset(dataset_path, model_type)
+    else:
+        raise ValueError("Model type must be one of: image, text, video, audio.")
+    if len(dataset) == 0:
+        raise ValueError(f"No {model_type} dataset files found under '{os.path.abspath(dataset_path)}'.")
+    print("Loaded dataset with {} samples from '{}'.".format(len(dataset), os.path.abspath(dataset_path)))
+    return dataset
+
+
+
+# Split the dataset into 2 parts - training and testing datasets
+def split_dataset(dataset, train_ratio):
+    if dataset is None:
+        raise ValueError("Dataset is None. Please provide a valid dataset.")
+    if isinstance(train_ratio, bool) or not isinstance(train_ratio, (int, float)) or not 2 <= train_ratio <= 98:
+        raise ValueError("Train ratio must be between 2 and 98.")
+    dataset_size = len(dataset)
+    if dataset_size < 2:
+        raise ValueError("At least two dataset samples are required to create non-empty train and test splits.")
+    train_size = int(train_ratio / 100 * dataset_size)
+    test_size = dataset_size - train_size
+    train_dataset, test_dataset = torch.utils.data.random_split(dataset, [train_size, test_size])
+    print("Dataset split into {} training samples and {} testing samples.".format(len(train_dataset), len(test_dataset)))
+    return train_dataset, test_dataset
+
+
+def data_loader(dataset, batch_size, shuffle):
+    if dataset is None:
+        raise ValueError("Dataset is None. Please provide a valid dataset.")
+    if batch_size <= 0:
+        raise ValueError("Batch size must be greater than 0.")
+    if not isinstance(shuffle, bool):
+        raise ValueError("Shuffle must be a boolean value.")
+    data_loader = DataLoader(dataset, batch_size=batch_size, shuffle=shuffle)
+    print("Data loader created with batch size {} and shuffle setting {}.".format(batch_size, shuffle))
+    return data_loader
+
+
+def dataset_convert(dataset_path, out_path, convert_type, model_type):
+    if dataset_path is None or out_path is None or convert_type is None or model_type is None:
+        raise ValueError("Dataset path, output path, type, and model type must not be None.")
+    if not os.path.exists(dataset_path):
+        raise ValueError("Dataset path does not exist.")
+    if model_type not in {"image", "text", "video", "audio"}:
+        raise ValueError("Model type must be one of: image, text, video, audio.")
+    if convert_type not in ['json', 'csv', 'tsv', 'xml', 'parquet', 'orc', 'feather', 'hdf5', 'pickle']:
+        raise ValueError("Convert type must be one of the following: 'json', 'csv', 'tsv', 'xml', 'parquet', 'orc', 'feather', 'hdf5', 'pickle'.")
+    output_parent = os.path.dirname(os.path.abspath(os.fspath(out_path)))
+    os.makedirs(output_parent, exist_ok=True)
+    if model_type == "image":
+        if convert_type == 'json':
+            dataset = datasets.ImageFolder(dataset_path, transform=v2.ToTensor())
+            data_loader = DataLoader(dataset, batch_size=1, shuffle=False)
+            data_list = []
+            for images, labels in data_loader:
+                for i in range(len(images)):
+                    image = images[i].numpy().tolist()
+                    label = labels[i].item()
+                    data_list.append({'image': image, 'label': label})
+            with open(out_path, 'w') as f:
+                json.dump(data_list, f)
+
+        elif convert_type == 'csv':
+            dataset = datasets.ImageFolder(dataset_path, transform=v2.ToTensor())
+            data_loader = DataLoader(dataset, batch_size=1, shuffle=False)
+            with open(out_path, 'w', newline='') as f:
+                writer = csv.writer(f)
+                writer.writerow(['image', 'label'])
+                for images, labels in data_loader:
+                    for i in range(len(images)):
+                        image = images[i].numpy().flatten().tolist()
+                        label = labels[i].item()
+                        writer.writerow([image, label])
+                f.flush()
+        elif convert_type == 'tsv':
+            dataset = datasets.ImageFolder(dataset_path, transform=v2.ToTensor())
+            data_loader = DataLoader(dataset, batch_size=1, shuffle=False)
+            with open(out_path, 'w', newline='') as f:
+                writer = csv.writer(f, delimiter='\t')
+                writer.writerow(['image', 'label'])
+                for images, labels in data_loader:
+                    for i in range(len(images)):
+                        image = images[i].numpy().flatten().tolist()
+                        label = labels[i].item()
+                        writer.writerow([image, label])
+                f.flush()
+        elif convert_type == 'xml':
+            dataset = datasets.ImageFolder(dataset_path, transform=v2.ToTensor())
+            data_loader = DataLoader(dataset, batch_size=1, shuffle=False)
+            with open(out_path, 'w') as f:
+                f.write('<dataset>\n')
+                for images, labels in data_loader:
+                    for i in range(len(images)):
+                        image = images[i].numpy().flatten().tolist()
+                        label = labels[i].item()
+                        f.write(f'  <data>\n    <image>{image}</image>\n    <label>{label}</label>\n  </data>\n')
+                f.write('</dataset>')
+        elif convert_type == 'parquet':
+            dataset = datasets.ImageFolder(dataset_path, transform=v2.ToTensor())
+            data_loader = DataLoader(dataset, batch_size=1, shuffle=False)
+            data_list = []
+            for images, labels in data_loader:
+                for i in range(len(images)):
+                    image = images[i].numpy().flatten().tolist()
+                    label = labels[i].item()
+                    data_list.append({'image': image, 'label': label})
+            df = pd.DataFrame(data_list)
+            df.to_parquet(out_path, index=False)
+
+        elif convert_type == 'orc':
+            dataset = datasets.ImageFolder(dataset_path, transform=v2.ToTensor())
+            data_loader = DataLoader(dataset, batch_size=1, shuffle=False)
+            data_list = []
+            for images, labels in data_loader:
+                for i in range(len(images)):
+                    image = images[i].numpy().flatten().tolist()
+                    label = labels[i].item()
+                    data_list.append({'image': image, 'label': label})
+            df = pd.DataFrame(data_list)
+            df.to_orc(out_path, index=False)
+
+        elif convert_type == 'feather':
+            dataset = datasets.ImageFolder(dataset_path, transform=v2.ToTensor())
+            data_loader = DataLoader(dataset, batch_size=1, shuffle=False)
+            data_list = []
+            for images, labels in data_loader:
+                for i in range(len(images)):
+                    image = images[i].numpy().flatten().tolist()
+                    label = labels[i].item()
+                    data_list.append({'image': image, 'label': label})
+            df = pd.DataFrame(data_list)
+            df.to_feather(out_path)
+
+        elif convert_type == 'hdf5':
+            dataset = datasets.ImageFolder(dataset_path, transform=v2.ToTensor())
+            data_loader = DataLoader(dataset, batch_size=1, shuffle=False)
+            data_list = []
+            for images, labels in data_loader:
+                for i in range(len(images)):
+                    image = images[i].numpy().flatten().tolist()
+                    label = labels[i].item()
+                    data_list.append({'image': image, 'label': label})
+            df = pd.DataFrame(data_list)
+            df.to_hdf(out_path, key='data', mode='w')
+
+        elif convert_type == 'pickle':
+            dataset = datasets.ImageFolder(dataset_path, transform=v2.ToTensor())
+            data_loader = DataLoader(dataset, batch_size=1, shuffle=False)
+            data_list = []
+            for images, labels in data_loader:
+                for i in range(len(images)):
+                    image = images[i].numpy().flatten().tolist()
+                    label = labels[i].item()
+                    data_list.append({'image': image, 'label': label})
+            df = pd.DataFrame(data_list)
+            df.to_pickle(out_path)  
+
+    if model_type == "text":
+        if convert_type == 'json':
+            dataset = _FolderDataset(dataset_path, "text")
+            data_loader = DataLoader(dataset, batch_size=1, shuffle=False)
+            data_list = []
+            for texts, labels in data_loader:
+                for i in range(len(texts)):
+                    text = texts[i]
+                    label = labels[i].item()
+                    data_list.append({'text': text, 'label': label})
+            with open(out_path, 'w') as f:
+                json.dump(data_list, f)
+        elif convert_type == 'csv':
+            dataset = _FolderDataset(dataset_path, "text")
+            data_loader = DataLoader(dataset, batch_size=1, shuffle=False)
+            with open(out_path, 'w', newline='') as f:
+                writer = csv.writer(f)
+                writer.writerow(['text', 'label'])
+                for texts, labels in data_loader:
+                    for i in range(len(texts)):
+                        text = texts[i]
+                        label = labels[i].item()
+                        writer.writerow([text, label])
+        elif convert_type == 'tsv':
+            dataset = _FolderDataset(dataset_path, "text")
+            data_loader = DataLoader(dataset, batch_size=1, shuffle=False)
+            with open(out_path, 'w', newline='') as f:
+                writer = csv.writer(f, delimiter='\t')
+                writer.writerow(['text', 'label'])
+                for texts, labels in data_loader:
+                    for i in range(len(texts)):
+                        text = texts[i]
+                        label = labels[i].item()
+                        writer.writerow([text, label])
+        elif convert_type == 'xml':
+            dataset = _FolderDataset(dataset_path, "text")
+            data_loader = DataLoader(dataset, batch_size=1, shuffle=False)
+            with open(out_path, 'w') as f:
+                f.write('<dataset>\n')
+                for texts, labels in data_loader:
+                    for i in range(len(texts)):
+                        text = texts[i]
+                        label = labels[i].item()
+                        f.write(f'  <data>\n    <text>{text}</text>\n    <label>{label}</label>\n  </data>\n')
+                f.write('</dataset>')
+        elif convert_type == 'parquet':
+            dataset = _FolderDataset(dataset_path, "text")
+            data_loader = DataLoader(dataset, batch_size=1, shuffle=False)
+            data_list = []
+            for texts, labels in data_loader:
+                for i in range(len(texts)):
+                    text = texts[i]
+                    label = labels[i].item()
+                    data_list.append({'text': text, 'label': label})
+            df = pd.DataFrame(data_list)
+            df.to_parquet(out_path, index=False)
+        elif convert_type == 'orc':
+            dataset = _FolderDataset(dataset_path, "text")
+            data_loader = DataLoader(dataset, batch_size=1, shuffle=False)
+            data_list = []
+            for texts, labels in data_loader:
+                for i in range(len(texts)):
+                    text = texts[i]
+                    label = labels[i].item()
+                    data_list.append({'text': text, 'label': label})
+            df = pd.DataFrame(data_list)
+            df.to_orc(out_path, index=False)
+        elif convert_type == 'feather':
+            dataset = _FolderDataset(dataset_path, "text")
+            data_loader = DataLoader(dataset, batch_size=1, shuffle=False)
+            data_list = []
+            for texts, labels in data_loader:
+                for i in range(len(texts)):
+                    text = texts[i]
+                    label = labels[i].item()
+                    data_list.append({'text': text, 'label': label})
+            df = pd.DataFrame(data_list)
+            df.to_feather(out_path)
+        elif convert_type == 'hdf5':
+            dataset = _FolderDataset(dataset_path, "text")
+            data_loader = DataLoader(dataset, batch_size=1, shuffle=False)
+            data_list = []
+            for texts, labels in data_loader:
+                for i in range(len(texts)):
+                    text = texts[i]
+                    label = labels[i].item()
+                    data_list.append({'text': text, 'label': label})
+            df = pd.DataFrame(data_list)
+            df.to_hdf(out_path, key='data', mode='w')
+        elif convert_type == 'pickle':
+            dataset = _FolderDataset(dataset_path, "text")
+            data_loader = DataLoader(dataset, batch_size=1, shuffle=False)
+            data_list = []
+            for texts, labels in data_loader:
+                for i in range(len(texts)):
+                    text = texts[i]
+                    label = labels[i].item()
+                    data_list.append({'text': text, 'label': label})
+            df = pd.DataFrame(data_list)
+            df.to_pickle(out_path)
+
+    if model_type == "video":
+        if convert_type == 'json':
+            dataset = _FolderDataset(dataset_path, "video")
+            data_loader = DataLoader(dataset, batch_size=1, shuffle=False)
+            data_list = []
+            for videos, labels in data_loader:
+                for i in range(len(videos)):
+                    video = videos[i]
+                    label = labels[i].item()
+                    data_list.append({'video': video, 'label': label})
+            with open(out_path, 'w') as f:
+                json.dump(data_list, f)
+
+        elif convert_type == 'csv':
+            dataset = _FolderDataset(dataset_path, "video")
+            data_loader = DataLoader(dataset, batch_size=1, shuffle=False)
+            with open(out_path, 'w', newline='') as f:
+                writer = csv.writer(f)
+                writer.writerow(['video', 'label'])
+                for videos, labels in data_loader:
+                    for i in range(len(videos)):
+                        video = videos[i]
+                        label = labels[i].item()
+                        writer.writerow([video, label])
+        elif convert_type == 'tsv':
+            dataset = _FolderDataset(dataset_path, "video")
+            data_loader = DataLoader(dataset, batch_size=1, shuffle=False)
+            with open(out_path, 'w', newline='') as f:
+                writer = csv.writer(f, delimiter='\t')
+                writer.writerow(['video', 'label'])
+                for videos, labels in data_loader:
+                    for i in range(len(videos)):
+                        video = videos[i]
+                        label = labels[i].item()
+                        writer.writerow([video, label])
+
+        elif convert_type == 'xml':
+            dataset = _FolderDataset(dataset_path, "video")
+            data_loader = DataLoader(dataset, batch_size=1, shuffle=False)
+            with open(out_path, 'w') as f:
+                f.write('<dataset>\n')
+                for videos, labels in data_loader:
+                    for i in range(len(videos)):
+                        video = videos[i]
+                        label = labels[i].item()
+                        f.write(f'  <data>\n    <video>{video}</video>\n    <label>{label}</label>\n  </data>\n')
+                f.write('</dataset>')
+
+        elif convert_type == 'parquet':
+            dataset = _FolderDataset(dataset_path, "video")
+            data_loader = DataLoader(dataset, batch_size=1, shuffle=False)
+            data_list = []
+            for videos, labels in data_loader:
+                for i in range(len(videos)):
+                    video = videos[i]
+                    label = labels[i].item()
+                    data_list.append({'video': video, 'label': label})
+            df = pd.DataFrame(data_list)
+            df.to_parquet(out_path, index=False)
+
+        elif convert_type == 'orc':
+            dataset = _FolderDataset(dataset_path, "video")
+            data_loader = DataLoader(dataset, batch_size=1, shuffle=False)
+            data_list = []
+            for videos, labels in data_loader:
+                for i in range(len(videos)):
+                    video = videos[i]
+                    label = labels[i].item()
+                    data_list.append({'video': video, 'label': label})
+            df = pd.DataFrame(data_list)
+            df.to_orc(out_path, index=False)
+
+        elif convert_type == 'feather':
+            dataset = _FolderDataset(dataset_path, "video")
+            data_loader = DataLoader(dataset, batch_size=1, shuffle=False)
+            data_list = []
+            for videos, labels in data_loader:
+                for i in range(len(videos)):
+                    video = videos[i]
+                    label = labels[i].item()
+                    data_list.append({'video': video, 'label': label})
+            df = pd.DataFrame(data_list)
+            df.to_feather(out_path)
+
+        elif convert_type == 'hdf5':
+            dataset = _FolderDataset(dataset_path, "video")
+            data_loader = DataLoader(dataset, batch_size=1, shuffle=False)
+            data_list = []
+            for videos, labels in data_loader:
+                for i in range(len(videos)):
+                    video = videos[i]
+                    label = labels[i].item()
+                    data_list.append({'video': video, 'label': label})
+            df = pd.DataFrame(data_list)
+            df.to_hdf(out_path, key='data', mode='w')
+
+        elif convert_type == 'pickle':
+            dataset = _FolderDataset(dataset_path, "video")
+            data_loader = DataLoader(dataset, batch_size=1, shuffle=False)
+            data_list = []
+            for videos, labels in data_loader:
+                for i in range(len(videos)):
+                    video = videos[i]
+                    label = labels[i].item()
+                    data_list.append({'video': video, 'label': label})
+            df = pd.DataFrame(data_list)
+            df.to_pickle(out_path)
+
+    if model_type == "audio":
+        if convert_type == 'json':
+            dataset = _FolderDataset(dataset_path, "audio")
+            data_loader = DataLoader(dataset, batch_size=1, shuffle=False)
+            data_list = []
+            for audios, labels in data_loader:
+                for i in range(len(audios)):
+                    audio = audios[i]
+                    label = labels[i].item()
+                    data_list.append({'audio': audio, 'label': label})
+            with open(out_path, 'w') as f:
+                json.dump(data_list, f)
+
+        elif convert_type == 'csv':
+            dataset = _FolderDataset(dataset_path, "audio")
+            data_loader = DataLoader(dataset, batch_size=1, shuffle=False)
+            with open(out_path, 'w', newline='') as f:
+                writer = csv.writer(f)
+                writer.writerow(['audio', 'label'])
+                for audios, labels in data_loader:
+                    for i in range(len(audios)):
+                        audio = audios[i]
+                        label = labels[i].item()
+                        writer.writerow([audio, label])
+        elif convert_type == 'tsv':
+            dataset = _FolderDataset(dataset_path, "audio")
+            data_loader = DataLoader(dataset, batch_size=1, shuffle=False)
+            with open(out_path, 'w', newline='') as f:
+                writer = csv.writer(f, delimiter='\t')
+                writer.writerow(['audio', 'label'])
+                for audios, labels in data_loader:
+                    for i in range(len(audios)):
+                        audio = audios[i]
+                        label = labels[i].item()
+                        writer.writerow([audio, label])
+        elif convert_type == 'xml':
+            dataset = _FolderDataset(dataset_path, "audio")
+            data_loader = DataLoader(dataset, batch_size=1, shuffle=False)
+            with open(out_path, 'w') as f:
+                f.write('<dataset>\n')
+                for audios, labels in data_loader:
+                    for i in range(len(audios)):
+                        audio = audios[i]
+                        label = labels[i].item()
+                        f.write(f'  <data>\n    <audio>{audio}</audio>\n    <label>{label}</label>\n  </data>\n')
+                f.write('</dataset>')
+
+        elif convert_type == 'parquet':
+            dataset = _FolderDataset(dataset_path, "audio")
+            data_loader = DataLoader(dataset, batch_size=1, shuffle=False)
+            data_list = []
+            for audios, labels in data_loader:
+                for i in range(len(audios)):
+                    audio = audios[i]
+                    label = labels[i].item()
+                    data_list.append({'audio': audio, 'label': label})
+            df = pd.DataFrame(data_list)
+            df.to_parquet(out_path, index=False)
+
+        
+        elif convert_type == 'orc':
+            dataset = _FolderDataset(dataset_path, "audio")
+            data_loader = DataLoader(dataset, batch_size=1, shuffle=False)
+            data_list = []
+            for audios, labels in data_loader:
+                for i in range(len(audios)):
+                    audio = audios[i]
+                    label = labels[i].item()
+                    data_list.append({'audio': audio, 'label': label})
+            df = pd.DataFrame(data_list)
+            df.to_orc(out_path, index=False)
+
+
+        elif convert_type == 'feather':
+            dataset = _FolderDataset(dataset_path, "audio")
+            data_loader = DataLoader(dataset, batch_size=1, shuffle=False)
+            data_list = []
+            for audios, labels in data_loader:
+                for i in range(len(audios)):
+                    audio = audios[i]
+                    label = labels[i].item()
+                    data_list.append({'audio': audio, 'label': label})
+            df = pd.DataFrame(data_list)
+            df.to_feather(out_path)
+
+        elif convert_type == 'hdf5':
+            dataset = _FolderDataset(dataset_path, "audio")
+            data_loader = DataLoader(dataset, batch_size=1, shuffle=False)
+            data_list = []
+            for audios, labels in data_loader:
+                for i in range(len(audios)):
+                    audio = audios[i]
+                    label = labels[i].item()
+                    data_list.append({'audio': audio, 'label': label})
+            df = pd.DataFrame(data_list)
+            df.to_hdf(out_path, key='data', mode='w')
+        
+
+        elif convert_type == 'pickle':
+            dataset = _FolderDataset(dataset_path, "audio")
+            data_loader = DataLoader(dataset, batch_size=1, shuffle=False)
+            data_list = []
+            for audios, labels in data_loader:
+                for i in range(len(audios)):
+                    audio = audios[i]
+                    label = labels[i].item()
+                    data_list.append({'audio': audio, 'label': label})
+            df = pd.DataFrame(data_list)
+            df.to_pickle(out_path)
+
+    output_path = os.path.abspath(os.fspath(out_path))
+    print(f"Converted {model_type} dataset to '{output_path}'.")
+    return output_path