alp-data 1.10.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- alp_data/__init__.py +103 -0
- alp_data/backends/__init__.py +25 -0
- alp_data/backends/backends.py +51 -0
- alp_data/backends/pandas_backend.py +1042 -0
- alp_data/backends/polars_backend.py +1219 -0
- alp_data/backends/protocol.py +805 -0
- alp_data/chain.py +179 -0
- alp_data/concat.py +541 -0
- alp_data/dataset.py +726 -0
- alp_data/datasets/__init__.py +76 -0
- alp_data/datasets/animal_sound_archive.py +367 -0
- alp_data/datasets/animalspeak.py +267 -0
- alp_data/datasets/anuraset.py +279 -0
- alp_data/datasets/arctic_bird_sounds.py +291 -0
- alp_data/datasets/audioset.py +430 -0
- alp_data/datasets/audioset_strong.py +317 -0
- alp_data/datasets/beans.py +310 -0
- alp_data/datasets/beans_zero.py +349 -0
- alp_data/datasets/bengalese_finch_calls.py +325 -0
- alp_data/datasets/birdeep.py +286 -0
- alp_data/datasets/birdset.py +285 -0
- alp_data/datasets/chiffchaff_id.py +265 -0
- alp_data/datasets/corvid_wascher.py +240 -0
- alp_data/datasets/dclde2026.py +448 -0
- alp_data/datasets/dinardo_dolphin_whistles.py +270 -0
- alp_data/datasets/esp_raincoast.py +315 -0
- alp_data/datasets/geladas.py +277 -0
- alp_data/datasets/giant_otters.py +262 -0
- alp_data/datasets/gibbon_solos.py +282 -0
- alp_data/datasets/hawaiian_birds.py +277 -0
- alp_data/datasets/inaturalist.py +392 -0
- alp_data/datasets/infant_marmosets_vox.py +349 -0
- alp_data/datasets/insectset_459.py +251 -0
- alp_data/datasets/littleowl_id.py +213 -0
- alp_data/datasets/macaques_coo_calls.py +249 -0
- alp_data/datasets/nocturnal_bird_migration.py +304 -0
- alp_data/datasets/pipit_id.py +218 -0
- alp_data/datasets/powdermill.py +267 -0
- alp_data/datasets/subsegmentation.py +263 -0
- alp_data/datasets/superb_starling.py +230 -0
- alp_data/datasets/voxaboxen.py +1224 -0
- alp_data/datasets/wabad.py +396 -0
- alp_data/datasets/watkins.py +276 -0
- alp_data/datasets/xeno_canto.py +407 -0
- alp_data/datasets/xeno_canto_annotated_jeantet_23.py +274 -0
- alp_data/datasets/zebra_finch_julie_elie.py +261 -0
- alp_data/discover/__init__.py +3 -0
- alp_data/discover/gbif_taxonomy.py +575 -0
- alp_data/io/__init__.py +30 -0
- alp_data/io/file_utils.py +42 -0
- alp_data/io/filesystem.py +116 -0
- alp_data/io/paths.py +806 -0
- alp_data/io/read_utils.py +487 -0
- alp_data/transforms/__init__.py +35 -0
- alp_data/transforms/balanced_sample.py +167 -0
- alp_data/transforms/deduplicate.py +83 -0
- alp_data/transforms/filter.py +88 -0
- alp_data/transforms/label_from_feature.py +128 -0
- alp_data/transforms/long_tail_upsample.py +168 -0
- alp_data/transforms/multilabel_from_features.py +126 -0
- alp_data/transforms/registry.py +100 -0
- alp_data/transforms/select_columns.py +92 -0
- alp_data/transforms/subsample.py +115 -0
- alp_data/utils.py +451 -0
- alp_data-1.10.0.dist-info/METADATA +159 -0
- alp_data-1.10.0.dist-info/RECORD +68 -0
- alp_data-1.10.0.dist-info/WHEEL +4 -0
- alp_data-1.10.0.dist-info/licenses/LICENSE +21 -0
alp_data/__init__.py
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
from .chain import ChainedDataset
|
|
2
|
+
from .concat import ConcatenatedDataset
|
|
3
|
+
from .dataset import (
|
|
4
|
+
ChainedDatasetConfig,
|
|
5
|
+
ConcatConfig,
|
|
6
|
+
Dataset,
|
|
7
|
+
DatasetConfig,
|
|
8
|
+
DatasetInfo,
|
|
9
|
+
dataset_class_from_name,
|
|
10
|
+
dataset_from_config,
|
|
11
|
+
list_registered_datasets,
|
|
12
|
+
print_registered_datasets,
|
|
13
|
+
register_config,
|
|
14
|
+
register_dataset,
|
|
15
|
+
)
|
|
16
|
+
from .datasets import (
|
|
17
|
+
DCLDE2026,
|
|
18
|
+
WABAD,
|
|
19
|
+
AnimalSoundArchive,
|
|
20
|
+
AnimalSpeak,
|
|
21
|
+
AnuraSetStrong,
|
|
22
|
+
ArcticBirdSounds,
|
|
23
|
+
AudioSet,
|
|
24
|
+
Beans,
|
|
25
|
+
BeansZero,
|
|
26
|
+
BengaleseFinchCalls,
|
|
27
|
+
Birdeep,
|
|
28
|
+
BirdSet,
|
|
29
|
+
ChiffchaffId,
|
|
30
|
+
DinardoDolphinWhistles,
|
|
31
|
+
ESPRaincoast,
|
|
32
|
+
Geladas,
|
|
33
|
+
GiantOtters,
|
|
34
|
+
GibbonSolos,
|
|
35
|
+
HawaiianBirds,
|
|
36
|
+
INaturalist,
|
|
37
|
+
InsectSet459,
|
|
38
|
+
LittleOwlId,
|
|
39
|
+
MacaquesCooCalls,
|
|
40
|
+
NocturnalBirdMigration,
|
|
41
|
+
PipitId,
|
|
42
|
+
Powdermill,
|
|
43
|
+
Subsegmentation,
|
|
44
|
+
SuperbStarling,
|
|
45
|
+
Voxaboxen,
|
|
46
|
+
VoxaboxenEvents,
|
|
47
|
+
XenoCanto,
|
|
48
|
+
XenoCantoAnnotatedJeantet23,
|
|
49
|
+
ZebraFinchJulieElie,
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
__all__ = [
|
|
53
|
+
"dataset_from_config",
|
|
54
|
+
"Dataset",
|
|
55
|
+
"DatasetInfo",
|
|
56
|
+
"DatasetConfig",
|
|
57
|
+
"ConcatConfig",
|
|
58
|
+
"ChainedDatasetConfig",
|
|
59
|
+
"list_registered_datasets",
|
|
60
|
+
"print_registered_datasets",
|
|
61
|
+
"dataset_class_from_name",
|
|
62
|
+
"register_dataset",
|
|
63
|
+
"register_config",
|
|
64
|
+
"dataset_class_from_name",
|
|
65
|
+
"ConcatenatedDataset",
|
|
66
|
+
"ChainedDataset",
|
|
67
|
+
"AnimalSoundArchive",
|
|
68
|
+
"AnimalSpeak",
|
|
69
|
+
"GiantOtters",
|
|
70
|
+
"INaturalist",
|
|
71
|
+
"InsectSet459",
|
|
72
|
+
"Beans",
|
|
73
|
+
"BeansZero",
|
|
74
|
+
"ZebraFinchJulieElie",
|
|
75
|
+
"BirdSet",
|
|
76
|
+
"BengaleseFinchCalls",
|
|
77
|
+
"ChiffchaffId",
|
|
78
|
+
"LittleOwlId",
|
|
79
|
+
"PipitId",
|
|
80
|
+
"AudioSet",
|
|
81
|
+
"MacaquesCooCalls",
|
|
82
|
+
"NocturnalBirdMigration",
|
|
83
|
+
"Subsegmentation",
|
|
84
|
+
"SuperbStarling",
|
|
85
|
+
"Voxaboxen",
|
|
86
|
+
"VoxaboxenEvents",
|
|
87
|
+
"WABAD",
|
|
88
|
+
"ESPRaincoast",
|
|
89
|
+
"Subsegmentation",
|
|
90
|
+
"WABAD",
|
|
91
|
+
"XenoCanto",
|
|
92
|
+
"XenoCantoAnnotatedJeantet23",
|
|
93
|
+
"HawaiianBirds",
|
|
94
|
+
"AnuraSetStrong",
|
|
95
|
+
"Powdermill",
|
|
96
|
+
"ArcticBirdSounds",
|
|
97
|
+
"NocturnalBirdMigration",
|
|
98
|
+
"Birdeep",
|
|
99
|
+
"Geladas",
|
|
100
|
+
"DCLDE2026",
|
|
101
|
+
"DinardoDolphinWhistles",
|
|
102
|
+
"GibbonSolos",
|
|
103
|
+
]
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"""Backend system for unified data operations across pandas, polars, and other libraries.
|
|
2
|
+
|
|
3
|
+
This module provides a protocol-based backend system that allows alp-data to work
|
|
4
|
+
with multiple data libraries through a common interface.
|
|
5
|
+
|
|
6
|
+
Two protocols are defined:
|
|
7
|
+
- StreamingDataBackend: For streaming-only data formats (e.g., WebDataset/tar files)
|
|
8
|
+
that only support iteration, not random access.
|
|
9
|
+
- DataBackend: For in-memory data formats (e.g., pandas, polars DataFrames)
|
|
10
|
+
that support both random access and iteration.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from .backends import BackendType, get_backend
|
|
14
|
+
from .pandas_backend import PandasBackend
|
|
15
|
+
from .polars_backend import PolarsBackend
|
|
16
|
+
from .protocol import DataBackend, StreamingDataBackend
|
|
17
|
+
|
|
18
|
+
__all__ = [
|
|
19
|
+
"DataBackend",
|
|
20
|
+
"StreamingDataBackend",
|
|
21
|
+
"PandasBackend",
|
|
22
|
+
"PolarsBackend",
|
|
23
|
+
"BackendType",
|
|
24
|
+
"get_backend",
|
|
25
|
+
]
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
"""Serves as a registry of sorts and type definitions for supported backends.
|
|
2
|
+
|
|
3
|
+
When more backends are added, they should be registered here.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from typing import Literal, Type
|
|
7
|
+
|
|
8
|
+
from alp_data.backends.pandas_backend import PandasBackend
|
|
9
|
+
from alp_data.backends.polars_backend import PolarsBackend
|
|
10
|
+
from alp_data.backends.protocol import DataBackend, StreamingDataBackend
|
|
11
|
+
|
|
12
|
+
BackendType = Literal["pandas", "polars"]
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
# We need to add new backends here when they are implemented
|
|
16
|
+
_BACKEND_REGISTRY: dict[str, Type[DataBackend | StreamingDataBackend]] = {
|
|
17
|
+
"pandas": PandasBackend,
|
|
18
|
+
"polars": PolarsBackend,
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def get_backend(backend: BackendType) -> Type[DataBackend | StreamingDataBackend]:
|
|
23
|
+
"""Get the backend class for the specified backend type.
|
|
24
|
+
|
|
25
|
+
Parameters
|
|
26
|
+
----------
|
|
27
|
+
backend : BackendType
|
|
28
|
+
Name of the backend ("pandas" or "polars")
|
|
29
|
+
|
|
30
|
+
Returns
|
|
31
|
+
-------
|
|
32
|
+
Type[DataBackend | StreamingDataBackend]
|
|
33
|
+
The backend class (not an instance)
|
|
34
|
+
|
|
35
|
+
Raises
|
|
36
|
+
------
|
|
37
|
+
ValueError
|
|
38
|
+
If the backend name is not recognized
|
|
39
|
+
|
|
40
|
+
Examples
|
|
41
|
+
--------
|
|
42
|
+
>>> backend_cls = get_backend("pandas")
|
|
43
|
+
>>> assert backend_cls is PandasBackend
|
|
44
|
+
>>> backend_cls = get_backend("polars")
|
|
45
|
+
>>> assert backend_cls is PolarsBackend
|
|
46
|
+
"""
|
|
47
|
+
if backend not in _BACKEND_REGISTRY:
|
|
48
|
+
raise ValueError(
|
|
49
|
+
f"Unknown backend: {backend}. Supported backends: {list(_BACKEND_REGISTRY.keys())}"
|
|
50
|
+
)
|
|
51
|
+
return _BACKEND_REGISTRY[backend]
|