acids-dataset 0.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- acids_dataset/__init__.py +98 -0
- acids_dataset/add_embedding.py +108 -0
- acids_dataset/cli.py +49 -0
- acids_dataset/configs/after.gin +22 -0
- acids_dataset/configs/default.gin +13 -0
- acids_dataset/configs/rave.gin +18 -0
- acids_dataset/datasets/__init__.py +2 -0
- acids_dataset/datasets/combined.py +117 -0
- acids_dataset/datasets/simple.py +365 -0
- acids_dataset/datasets/utils.py +121 -0
- acids_dataset/features/__init__.py +11 -0
- acids_dataset/features/base.py +172 -0
- acids_dataset/features/beat_this/__init__.py +0 -0
- acids_dataset/features/beat_this/cli.py +165 -0
- acids_dataset/features/beat_this/inference.py +317 -0
- acids_dataset/features/beat_this/model/__init__.py +0 -0
- acids_dataset/features/beat_this/model/beat_tracker.py +337 -0
- acids_dataset/features/beat_this/model/loss.py +160 -0
- acids_dataset/features/beat_this/model/pl_module.py +369 -0
- acids_dataset/features/beat_this/model/postprocessor.py +199 -0
- acids_dataset/features/beat_this/model/roformer.py +181 -0
- acids_dataset/features/beat_this/preprocessing.py +59 -0
- acids_dataset/features/beat_this/utils.py +89 -0
- acids_dataset/features/beat_tracking.py +171 -0
- acids_dataset/features/clustering.py +47 -0
- acids_dataset/features/cqt.py +72 -0
- acids_dataset/features/loudness.py +45 -0
- acids_dataset/features/mel.py +52 -0
- acids_dataset/features/midi.py +165 -0
- acids_dataset/features/module.py +149 -0
- acids_dataset/features/pitch.py +218 -0
- acids_dataset/features/regexp.py +97 -0
- acids_dataset/fragments/__init__.py +39 -0
- acids_dataset/fragments/acids.py +67 -0
- acids_dataset/fragments/base.py +291 -0
- acids_dataset/fragments/compiled/__init__.py +0 -0
- acids_dataset/fragments/compiled/acids_pb2.py +52 -0
- acids_dataset/fragments/utils.py +32 -0
- acids_dataset/info.py +73 -0
- acids_dataset/loaders/__init__.py +0 -0
- acids_dataset/loaders/balanced.py +102 -0
- acids_dataset/parsers/__init__.py +1 -0
- acids_dataset/parsers/raw_parser.py +235 -0
- acids_dataset/parsers/utils.py +10 -0
- acids_dataset/preprocess.py +133 -0
- acids_dataset/transforms/__init__.py +9 -0
- acids_dataset/transforms/audiomentations.py +94 -0
- acids_dataset/transforms/base.py +197 -0
- acids_dataset/transforms/basic_pitch_torch/__init__.py +0 -0
- acids_dataset/transforms/basic_pitch_torch/assets/basic_pitch_pytorch_icassp_2022.pth +0 -0
- acids_dataset/transforms/basic_pitch_torch/constants.py +53 -0
- acids_dataset/transforms/basic_pitch_torch/inference.py +390 -0
- acids_dataset/transforms/basic_pitch_torch/model.py +215 -0
- acids_dataset/transforms/basic_pitch_torch/note_creation.py +498 -0
- acids_dataset/transforms/misc.py +411 -0
- acids_dataset/transforms/pitch.py +72 -0
- acids_dataset/transforms/resample_poly.py +109 -0
- acids_dataset/transforms/utils.py +54 -0
- acids_dataset/update.py +85 -0
- acids_dataset/utils.py +362 -0
- acids_dataset/writers/__init__.py +10 -0
- acids_dataset/writers/lmdb_writer.py +659 -0
- acids_dataset/writers/utils.py +147 -0
- acids_dataset-0.0.0.dist-info/METADATA +564 -0
- acids_dataset-0.0.0.dist-info/RECORD +69 -0
- acids_dataset-0.0.0.dist-info/WHEEL +5 -0
- acids_dataset-0.0.0.dist-info/entry_points.txt +2 -0
- acids_dataset-0.0.0.dist-info/licenses/LICENSE +352 -0
- acids_dataset-0.0.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
import yaml
|
|
3
|
+
import gin
|
|
4
|
+
import os
|
|
5
|
+
gin.enter_interactive_mode()
|
|
6
|
+
|
|
7
|
+
import torch
|
|
8
|
+
import torchaudio
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
ACIDS_DATASET_CONFIG_PATH = Path(__file__).parent / "configs"
|
|
12
|
+
ACIDS_DATASET_CUSTOM_CONFIG_PATH = Path(os.getcwd()) / "ad_configs"
|
|
13
|
+
TRANSFORM_GIN_PATH = ACIDS_DATASET_CONFIG_PATH / "transforms"
|
|
14
|
+
FEATURES_GIN_PATH = ACIDS_DATASET_CONFIG_PATH / "features"
|
|
15
|
+
|
|
16
|
+
def dataset_config_paths():
|
|
17
|
+
return [ACIDS_DATASET_CONFIG_PATH, ACIDS_DATASET_CUSTOM_CONFIG_PATH]
|
|
18
|
+
|
|
19
|
+
def import_database_configs(*add_paths):
|
|
20
|
+
gin.add_config_file_search_path(str(ACIDS_DATASET_CONFIG_PATH))
|
|
21
|
+
gin.add_config_file_search_path(str(ACIDS_DATASET_CUSTOM_CONFIG_PATH))
|
|
22
|
+
for a in add_paths:
|
|
23
|
+
gin.add_config_file_search_path(str(a))
|
|
24
|
+
|
|
25
|
+
from . import utils
|
|
26
|
+
from . import transforms
|
|
27
|
+
|
|
28
|
+
transforms.check_transform_configs(transforms, TRANSFORM_GIN_PATH)
|
|
29
|
+
def get_transform_config_path():
|
|
30
|
+
return TRANSFORM_GIN_PATH
|
|
31
|
+
|
|
32
|
+
from . import parsers
|
|
33
|
+
from . import features
|
|
34
|
+
|
|
35
|
+
features.check_feature_configs(features, FEATURES_GIN_PATH)
|
|
36
|
+
def get_feature_config_paths():
|
|
37
|
+
return [FEATURES_GIN_PATH]
|
|
38
|
+
|
|
39
|
+
from . import fragments
|
|
40
|
+
|
|
41
|
+
def get_fragment_class(class_name):
|
|
42
|
+
obj = getattr(fragments, class_name, None)
|
|
43
|
+
if not issubclass(obj, fragments.AudioFragment):
|
|
44
|
+
raise TypeError('fragment class %s not valid'%class_name)
|
|
45
|
+
return obj
|
|
46
|
+
|
|
47
|
+
def get_parser_class(class_name):
|
|
48
|
+
obj = getattr(parsers, class_name, None)
|
|
49
|
+
# if not issubclass(obj, fragments.AudioFragment):
|
|
50
|
+
# raise TypeError('fragment class %s not valid'%class_name)
|
|
51
|
+
return obj
|
|
52
|
+
|
|
53
|
+
def get_metadata_from_path(dataset_path):
|
|
54
|
+
dataset_path = Path(dataset_path)
|
|
55
|
+
metadata_path = dataset_path / "metadata.yaml"
|
|
56
|
+
with open(metadata_path, "r") as f:
|
|
57
|
+
metadata = yaml.safe_load(f)
|
|
58
|
+
return metadata
|
|
59
|
+
|
|
60
|
+
def get_writer_from_metadata(metadata):
|
|
61
|
+
return getattr(writers, metadata.get('writer_class', 'LMDBWriter'))
|
|
62
|
+
|
|
63
|
+
def get_writer_class_from_path(path):
|
|
64
|
+
metadata = get_metadata_from_path(path)
|
|
65
|
+
return getattr(writers, metadata.get('writer_class', 'LMDBWriter'))
|
|
66
|
+
|
|
67
|
+
def get_env_from_path(path):
|
|
68
|
+
writer_class = get_writer_class_from_path(path)
|
|
69
|
+
return writer_class.open(path)
|
|
70
|
+
|
|
71
|
+
def get_feature_hash_from_path(path):
|
|
72
|
+
writer_class = get_writer_class_from_path(path)
|
|
73
|
+
return writer_class.get_feature_hash(path)
|
|
74
|
+
|
|
75
|
+
def get_fragment_class_from_path(path):
|
|
76
|
+
metadata = get_metadata_from_path(path)
|
|
77
|
+
return get_fragment_class(metadata['fragment_class'])
|
|
78
|
+
|
|
79
|
+
def get_parser_class_from_path(path):
|
|
80
|
+
metadata = get_metadata_from_path(path)
|
|
81
|
+
return get_parser_class(metadata['parser_class'])
|
|
82
|
+
|
|
83
|
+
def get_features_from_path(path):
|
|
84
|
+
writer_class = get_writer_class_from_path(path)
|
|
85
|
+
return writer_class.loader(path).features
|
|
86
|
+
|
|
87
|
+
def get_feature_names_from_path(path):
|
|
88
|
+
writer_class = get_writer_class_from_path(path)
|
|
89
|
+
return list(writer_class.loader(path).features.keys())
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
from . import writers
|
|
93
|
+
from . import datasets
|
|
94
|
+
|
|
95
|
+
from .preprocess import *
|
|
96
|
+
from .update import *
|
|
97
|
+
from .add_embedding import *
|
|
98
|
+
from .utils import GinEnv
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import gin
|
|
3
|
+
from typing import List
|
|
4
|
+
import torch, torch.nn as nn
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from absl import flags, app
|
|
7
|
+
|
|
8
|
+
import sys
|
|
9
|
+
import sys; sys.path.append(str(Path(__file__).parent.parent))
|
|
10
|
+
|
|
11
|
+
from acids_dataset import get_metadata_from_path, get_writer_class_from_path
|
|
12
|
+
from acids_dataset import writers, transforms as adt
|
|
13
|
+
from acids_dataset.features import ModuleEmbedding
|
|
14
|
+
from acids_dataset.transforms import Transform
|
|
15
|
+
from acids_dataset.utils import GinEnv, parse_features, transform_from_gin_config, set_gin_constant
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def import_flags():
|
|
19
|
+
flags.DEFINE_string('path', None, 'dataset path', required=True)
|
|
20
|
+
flags.DEFINE_string('module', None, 'path to module (.ts)', required=True)
|
|
21
|
+
flags.DEFINE_string('method', "forward", 'model method to call', required=False)
|
|
22
|
+
flags.DEFINE_string('name', None, 'embedding name (default : filename)', required=False)
|
|
23
|
+
flags.DEFINE_multi_string('transform', [], help='transforms for different embedding views.')
|
|
24
|
+
flags.DEFINE_integer('model_sr', None, 'original sample rate of target model', required=False)
|
|
25
|
+
flags.DEFINE_string('device', None, 'device id used for computation')
|
|
26
|
+
flags.DEFINE_boolean('overwrite', False, help="recomputes the feature if already present in the dataset, and overwrites existing files")
|
|
27
|
+
flags.DEFINE_boolean('check', True, help="recomputes the feature if already present in the dataset")
|
|
28
|
+
|
|
29
|
+
def add_embedding_to_dataset(
|
|
30
|
+
path,
|
|
31
|
+
module: nn.Module = None,
|
|
32
|
+
module_path: str | Path | None = None,
|
|
33
|
+
module_sr: int | None = None,
|
|
34
|
+
method: str = "forward",
|
|
35
|
+
transforms: List[str | adt.Transform] = [],
|
|
36
|
+
name: str | None = None,
|
|
37
|
+
device: str | None = None,
|
|
38
|
+
check: bool = False,
|
|
39
|
+
overwrite: bool = False,
|
|
40
|
+
):
|
|
41
|
+
|
|
42
|
+
path = Path(path)
|
|
43
|
+
if name is None:
|
|
44
|
+
if module is None:
|
|
45
|
+
name = name or Path(module_path).stem
|
|
46
|
+
elif module_path is None:
|
|
47
|
+
name = name or type(module).__name__.capitalize()
|
|
48
|
+
else:
|
|
49
|
+
raise ValueError('either module or module_path must be given')
|
|
50
|
+
|
|
51
|
+
# parse gin constants
|
|
52
|
+
gin.add_config_file_search_path(Path(__file__).parent / "configs")
|
|
53
|
+
gin.add_config_file_search_path(path)
|
|
54
|
+
metadata = get_metadata_from_path(path)
|
|
55
|
+
set_gin_constant('SAMPLE_RATE', metadata['sr'])
|
|
56
|
+
set_gin_constant('CHANNELS', metadata['channels'])
|
|
57
|
+
set_gin_constant('DEVICE', device)
|
|
58
|
+
|
|
59
|
+
operative_transforms = []
|
|
60
|
+
for t in transforms:
|
|
61
|
+
if isinstance(t, str):
|
|
62
|
+
operative_transforms.append(transform_from_gin_config(t))
|
|
63
|
+
elif isinstance(t, Transform):
|
|
64
|
+
operative_transforms.append(t)
|
|
65
|
+
|
|
66
|
+
print(transforms, operative_transforms)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
# parse module
|
|
70
|
+
module_feature = ModuleEmbedding(module=module,
|
|
71
|
+
module_path=module_path,
|
|
72
|
+
module_sr=module_sr,
|
|
73
|
+
method=method,
|
|
74
|
+
transforms=operative_transforms,
|
|
75
|
+
sr=metadata['sr'],
|
|
76
|
+
name=name,
|
|
77
|
+
device=device)
|
|
78
|
+
|
|
79
|
+
# build writer
|
|
80
|
+
writer_class = get_writer_class_from_path(path)
|
|
81
|
+
writer_class.update(
|
|
82
|
+
path,
|
|
83
|
+
features=[module_feature],
|
|
84
|
+
check=check,
|
|
85
|
+
overwrite=overwrite
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def main(argv):
|
|
90
|
+
FLAGS = flags.FLAGS
|
|
91
|
+
add_embedding_to_dataset(
|
|
92
|
+
FLAGS.path,
|
|
93
|
+
module_path=FLAGS.module,
|
|
94
|
+
module_sr=FLAGS.model_sr,
|
|
95
|
+
method=FLAGS.method,
|
|
96
|
+
transforms=FLAGS.transform,
|
|
97
|
+
name=FLAGS.name,
|
|
98
|
+
device=FLAGS.device,
|
|
99
|
+
check=FLAGS.check,
|
|
100
|
+
overwrite=FLAGS.overwrite
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
if __name__ == "__main__":
|
|
105
|
+
app.run(main)
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
__name__ = "add_embedding"
|
acids_dataset/cli.py
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
from absl import app
|
|
3
|
+
|
|
4
|
+
AVAILABLE_SCRIPTS = [
|
|
5
|
+
'preprocess',
|
|
6
|
+
'update',
|
|
7
|
+
'info',
|
|
8
|
+
'add_embedding'
|
|
9
|
+
]
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def help():
|
|
13
|
+
print(f"""usage: acids-dataset [ {' | '.join(AVAILABLE_SCRIPTS)} ]
|
|
14
|
+
|
|
15
|
+
positional arguments:
|
|
16
|
+
command Command to launch with acids-dataset. valid commands : {' | '.join(AVAILABLE_SCRIPTS)}
|
|
17
|
+
""")
|
|
18
|
+
exit()
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def main():
|
|
22
|
+
if len(sys.argv) == 1:
|
|
23
|
+
help()
|
|
24
|
+
elif sys.argv[1] not in AVAILABLE_SCRIPTS:
|
|
25
|
+
help()
|
|
26
|
+
|
|
27
|
+
command = sys.argv[1]
|
|
28
|
+
|
|
29
|
+
if command == 'preprocess':
|
|
30
|
+
from acids_dataset import preprocess
|
|
31
|
+
preprocess.import_flags()
|
|
32
|
+
sys.argv[0] = preprocess.__name__
|
|
33
|
+
app.run(preprocess.main)
|
|
34
|
+
if command == 'update':
|
|
35
|
+
from acids_dataset import update
|
|
36
|
+
update.import_flags()
|
|
37
|
+
sys.argv[0] = update.__name__
|
|
38
|
+
app.run(update.main)
|
|
39
|
+
elif command == 'info':
|
|
40
|
+
from acids_dataset import info
|
|
41
|
+
info.import_flags()
|
|
42
|
+
sys.argv[0] = info.__name__
|
|
43
|
+
app.run(info.main)
|
|
44
|
+
elif command == "add_embedding":
|
|
45
|
+
from acids_dataset import add_embedding
|
|
46
|
+
add_embedding.import_flags()
|
|
47
|
+
sys.argv[0] = add_embedding.__name__
|
|
48
|
+
app.run(add_embedding.main)
|
|
49
|
+
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
CHUNK_LENGTH = 524288
|
|
2
|
+
HOP_LENGTH = 262144
|
|
3
|
+
SAMPLE_RATE = 44100
|
|
4
|
+
CHANNELS = 1
|
|
5
|
+
|
|
6
|
+
RawParser:
|
|
7
|
+
chunk_length = 524288
|
|
8
|
+
hop_length = 262144
|
|
9
|
+
sr = %SAMPLE_RATE
|
|
10
|
+
channels = %CHANNELS
|
|
11
|
+
pad_mode = "discard"
|
|
12
|
+
loudness_threshold = -60
|
|
13
|
+
|
|
14
|
+
LMDBWriter:
|
|
15
|
+
fragment_class = @AcidsFragment
|
|
16
|
+
parser = @RawParser
|
|
17
|
+
|
|
18
|
+
#parse_features:
|
|
19
|
+
# features = [@features.AfterMIDI]
|
|
20
|
+
# device = %DEVICE
|
|
21
|
+
|
|
22
|
+
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
CHUNK_LENGTH=262144
|
|
2
|
+
HOP_LENGTH=131072
|
|
3
|
+
SAMPLE_RATE = 44100
|
|
4
|
+
CHANNELS = 1
|
|
5
|
+
|
|
6
|
+
RawParser:
|
|
7
|
+
chunk_length = %CHUNK_LENGTH
|
|
8
|
+
hop_length = %HOP_LENGTH
|
|
9
|
+
sr = %SAMPLE_RATE
|
|
10
|
+
channels = %CHANNELS
|
|
11
|
+
pad_mode = "zero"
|
|
12
|
+
discard_if_lower_than = 32000
|
|
13
|
+
|
|
14
|
+
LMDBWriter:
|
|
15
|
+
fragment_class = @AcidsFragment
|
|
16
|
+
parser = @RawParser
|
|
17
|
+
|
|
18
|
+
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
from typing import List
|
|
2
|
+
import bisect
|
|
3
|
+
import math
|
|
4
|
+
import copy
|
|
5
|
+
from absl import logging
|
|
6
|
+
import torch
|
|
7
|
+
from functools import reduce
|
|
8
|
+
from .simple import AudioDataset
|
|
9
|
+
from ..utils import checklist
|
|
10
|
+
|
|
11
|
+
def cumsum(l):
|
|
12
|
+
return [sum(l[:i+1]) for i in range(len(l))]
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class CombinedAudioDataset(torch.utils.data.Dataset):
|
|
16
|
+
_fields_to_compare_ = ['_output_pattern', '_n_channels']
|
|
17
|
+
def __init__(self, datasets: List[AudioDataset], weights: List[float] | None = None, max_samples: int | None = None, use_cache: bool = False):
|
|
18
|
+
assert len(datasets) > 1, "CombinedAudioDataset must have at least 2 datasets as inputs"
|
|
19
|
+
self._compare_datasets(datasets)
|
|
20
|
+
if weights is None:
|
|
21
|
+
weights = [1 / len(datasets) ] * len(datasets)
|
|
22
|
+
assert len(weights) == len(datasets), "datasets and weights must be the same size"
|
|
23
|
+
self._weights = weights
|
|
24
|
+
self._datasets = datasets
|
|
25
|
+
self._dataset_map = cumsum([len(d) for d in self._datasets])
|
|
26
|
+
self._output_pattern = self._datasets[0]._output_pattern
|
|
27
|
+
self._n_channels = self._datasets[0]._n_channels
|
|
28
|
+
self._parent = None
|
|
29
|
+
self._subindices = None
|
|
30
|
+
self._max_samples = max_samples
|
|
31
|
+
self._partitions = {}
|
|
32
|
+
if use_cache: logging.warning("use_cache is not implemented yet in CombinedAudioDataset")
|
|
33
|
+
|
|
34
|
+
def _compare_datasets(self, datasets):
|
|
35
|
+
for field in self._fields_to_compare_:
|
|
36
|
+
assert len(set([getattr(d, field) for d in datasets])) == 1
|
|
37
|
+
|
|
38
|
+
def __getitem__(self, index):
|
|
39
|
+
target_dataset = bisect.bisect_right(self._dataset_map, index)
|
|
40
|
+
if self._subindices is None:
|
|
41
|
+
if target_dataset > 0: index -= self._dataset_map[target_dataset - 1]
|
|
42
|
+
else:
|
|
43
|
+
if target_dataset > 0: index -= self._dataset_map[target_dataset - 1]
|
|
44
|
+
index = self._subindices[target_dataset][index]
|
|
45
|
+
return self._datasets[target_dataset][index]
|
|
46
|
+
|
|
47
|
+
def __len__(self):
|
|
48
|
+
if self._subindices is None:
|
|
49
|
+
return sum([len(d) for d in self._datasets])
|
|
50
|
+
else:
|
|
51
|
+
return sum([len(s) for s in self._subindices])
|
|
52
|
+
|
|
53
|
+
def get_sampler(self, valid: bool = False, replacement: bool = False):
|
|
54
|
+
max_samples = len(self) if self._max_samples is None else self._max_samples
|
|
55
|
+
weights = sum([[self._weights[i]] * len(self._datasets[i]) for i in range(len(self._datasets))], [])
|
|
56
|
+
if valid:
|
|
57
|
+
return torch.utils.data.WeightedRandomSampler(weights, max_samples, replacement=replacement)
|
|
58
|
+
else:
|
|
59
|
+
return torch.utils.data.WeightedRandomSampler(weights, max_samples, replacement=replacement, generator=torch.Generator().manual_seed(42))
|
|
60
|
+
|
|
61
|
+
@property
|
|
62
|
+
def parent(self):
|
|
63
|
+
return self._parent
|
|
64
|
+
|
|
65
|
+
@parent.setter
|
|
66
|
+
def parent(self, obj):
|
|
67
|
+
assert issubclass(type(obj), type(self)), "parent of a dataset must be a subclass of %s"%(type(self))
|
|
68
|
+
|
|
69
|
+
@property
|
|
70
|
+
def keys(self) -> List[List[str]]:
|
|
71
|
+
return [d.keys for d in self._datasets]
|
|
72
|
+
|
|
73
|
+
def get_subdataset(self, subindices, check=False):
|
|
74
|
+
assert len(subindices) == len(self._datasets)
|
|
75
|
+
subdataset = copy.copy(self)
|
|
76
|
+
subdataset._subindices = subindices
|
|
77
|
+
subdataset._dataset_map = cumsum([len(d) for d in subdataset._subindices])
|
|
78
|
+
subdataset.parent = self
|
|
79
|
+
if self._max_samples:
|
|
80
|
+
subdataset._max_samples = int(self._max_samples * (len(subindices) / len(self)))
|
|
81
|
+
return subdataset
|
|
82
|
+
|
|
83
|
+
def _check_feature_in_subdatasets(self, features):
|
|
84
|
+
for f in checklist(features):
|
|
85
|
+
for i, d in enumerate(self._datasets):
|
|
86
|
+
assert f in d.features or f == "original_path", "feature %s not present in dataset %d (path : %s)"%(f, i, d._db_path)
|
|
87
|
+
|
|
88
|
+
def split(self,
|
|
89
|
+
partitions,
|
|
90
|
+
features = None,
|
|
91
|
+
**kwargs):
|
|
92
|
+
assert self.parent is None, "dataset is already a partition"
|
|
93
|
+
if features is not None:
|
|
94
|
+
self._check_feature_in_subdatasets(features)
|
|
95
|
+
partitions_list = [
|
|
96
|
+
d.split(partitions, **kwargs)
|
|
97
|
+
for d in self._datasets
|
|
98
|
+
]
|
|
99
|
+
partitions = {}
|
|
100
|
+
for p in partitions_list[0].keys():
|
|
101
|
+
partitions[p] = [d[p] for d in partitions_list]
|
|
102
|
+
return {p: self.get_subdataset(partitions[p]) for p in partitions.keys()}
|
|
103
|
+
|
|
104
|
+
def load_partition(self, name: str | None, check: bool = False):
|
|
105
|
+
partitions = {}
|
|
106
|
+
for i, d in enumerate(self._datasets):
|
|
107
|
+
dpart = d.load_partition(name, check=check)
|
|
108
|
+
if i == 0: partitions.update({k: [None] * len(self._datasets) for k in dpart.keys()})
|
|
109
|
+
for k in dpart.keys():
|
|
110
|
+
partitions[k][i] = dpart[k]._subindices
|
|
111
|
+
return {p: self.get_subdataset(partitions[p]) for p in partitions.keys()}
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
|