alp-data 1.10.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (236) hide show
  1. alp_data-1.10.0/.dict-allowed.txt +10 -0
  2. alp_data-1.10.0/.github/CODEOWNERS +1 -0
  3. alp_data-1.10.0/.github/workflows/ci.yml +66 -0
  4. alp_data-1.10.0/.github/workflows/docs-deploy.yml +58 -0
  5. alp_data-1.10.0/.github/workflows/docs-validate.yml +30 -0
  6. alp_data-1.10.0/.github/workflows/pre-commit.yml +16 -0
  7. alp_data-1.10.0/.github/workflows/publish_public.yml +35 -0
  8. alp_data-1.10.0/.gitignore +184 -0
  9. alp_data-1.10.0/.pre-commit-config.yaml +52 -0
  10. alp_data-1.10.0/.python-version +1 -0
  11. alp_data-1.10.0/.yamllint +6 -0
  12. alp_data-1.10.0/AGENTS.md +14 -0
  13. alp_data-1.10.0/CLAUDE.md +1 -0
  14. alp_data-1.10.0/LICENSE +21 -0
  15. alp_data-1.10.0/Makefile +8 -0
  16. alp_data-1.10.0/PKG-INFO +159 -0
  17. alp_data-1.10.0/README.md +138 -0
  18. alp_data-1.10.0/alp_data/__init__.py +103 -0
  19. alp_data-1.10.0/alp_data/backends/__init__.py +25 -0
  20. alp_data-1.10.0/alp_data/backends/backends.py +51 -0
  21. alp_data-1.10.0/alp_data/backends/pandas_backend.py +1042 -0
  22. alp_data-1.10.0/alp_data/backends/polars_backend.py +1219 -0
  23. alp_data-1.10.0/alp_data/backends/protocol.py +805 -0
  24. alp_data-1.10.0/alp_data/chain.py +179 -0
  25. alp_data-1.10.0/alp_data/concat.py +541 -0
  26. alp_data-1.10.0/alp_data/dataset.py +726 -0
  27. alp_data-1.10.0/alp_data/datasets/__init__.py +76 -0
  28. alp_data-1.10.0/alp_data/datasets/animal_sound_archive.py +367 -0
  29. alp_data-1.10.0/alp_data/datasets/animalspeak.py +267 -0
  30. alp_data-1.10.0/alp_data/datasets/anuraset.py +279 -0
  31. alp_data-1.10.0/alp_data/datasets/arctic_bird_sounds.py +291 -0
  32. alp_data-1.10.0/alp_data/datasets/audioset.py +430 -0
  33. alp_data-1.10.0/alp_data/datasets/audioset_strong.py +317 -0
  34. alp_data-1.10.0/alp_data/datasets/beans.py +310 -0
  35. alp_data-1.10.0/alp_data/datasets/beans_zero.py +349 -0
  36. alp_data-1.10.0/alp_data/datasets/bengalese_finch_calls.py +325 -0
  37. alp_data-1.10.0/alp_data/datasets/birdeep.py +286 -0
  38. alp_data-1.10.0/alp_data/datasets/birdset.py +285 -0
  39. alp_data-1.10.0/alp_data/datasets/chiffchaff_id.py +265 -0
  40. alp_data-1.10.0/alp_data/datasets/corvid_wascher.py +240 -0
  41. alp_data-1.10.0/alp_data/datasets/dclde2026.py +448 -0
  42. alp_data-1.10.0/alp_data/datasets/dinardo_dolphin_whistles.py +270 -0
  43. alp_data-1.10.0/alp_data/datasets/esp_raincoast.py +315 -0
  44. alp_data-1.10.0/alp_data/datasets/geladas.py +277 -0
  45. alp_data-1.10.0/alp_data/datasets/giant_otters.py +262 -0
  46. alp_data-1.10.0/alp_data/datasets/gibbon_solos.py +282 -0
  47. alp_data-1.10.0/alp_data/datasets/hawaiian_birds.py +277 -0
  48. alp_data-1.10.0/alp_data/datasets/inaturalist.py +392 -0
  49. alp_data-1.10.0/alp_data/datasets/infant_marmosets_vox.py +349 -0
  50. alp_data-1.10.0/alp_data/datasets/insectset_459.py +251 -0
  51. alp_data-1.10.0/alp_data/datasets/littleowl_id.py +213 -0
  52. alp_data-1.10.0/alp_data/datasets/macaques_coo_calls.py +249 -0
  53. alp_data-1.10.0/alp_data/datasets/nocturnal_bird_migration.py +304 -0
  54. alp_data-1.10.0/alp_data/datasets/pipit_id.py +218 -0
  55. alp_data-1.10.0/alp_data/datasets/powdermill.py +267 -0
  56. alp_data-1.10.0/alp_data/datasets/subsegmentation.py +263 -0
  57. alp_data-1.10.0/alp_data/datasets/superb_starling.py +230 -0
  58. alp_data-1.10.0/alp_data/datasets/voxaboxen.py +1224 -0
  59. alp_data-1.10.0/alp_data/datasets/wabad.py +396 -0
  60. alp_data-1.10.0/alp_data/datasets/watkins.py +276 -0
  61. alp_data-1.10.0/alp_data/datasets/xeno_canto.py +407 -0
  62. alp_data-1.10.0/alp_data/datasets/xeno_canto_annotated_jeantet_23.py +274 -0
  63. alp_data-1.10.0/alp_data/datasets/zebra_finch_julie_elie.py +261 -0
  64. alp_data-1.10.0/alp_data/discover/__init__.py +3 -0
  65. alp_data-1.10.0/alp_data/discover/gbif_taxonomy.py +575 -0
  66. alp_data-1.10.0/alp_data/io/__init__.py +30 -0
  67. alp_data-1.10.0/alp_data/io/file_utils.py +42 -0
  68. alp_data-1.10.0/alp_data/io/filesystem.py +116 -0
  69. alp_data-1.10.0/alp_data/io/paths.py +806 -0
  70. alp_data-1.10.0/alp_data/io/read_utils.py +487 -0
  71. alp_data-1.10.0/alp_data/transforms/__init__.py +35 -0
  72. alp_data-1.10.0/alp_data/transforms/balanced_sample.py +167 -0
  73. alp_data-1.10.0/alp_data/transforms/deduplicate.py +83 -0
  74. alp_data-1.10.0/alp_data/transforms/filter.py +88 -0
  75. alp_data-1.10.0/alp_data/transforms/label_from_feature.py +128 -0
  76. alp_data-1.10.0/alp_data/transforms/long_tail_upsample.py +168 -0
  77. alp_data-1.10.0/alp_data/transforms/multilabel_from_features.py +126 -0
  78. alp_data-1.10.0/alp_data/transforms/registry.py +100 -0
  79. alp_data-1.10.0/alp_data/transforms/select_columns.py +92 -0
  80. alp_data-1.10.0/alp_data/transforms/subsample.py +115 -0
  81. alp_data-1.10.0/alp_data/utils.py +451 -0
  82. alp_data-1.10.0/docs/assets/favicon.png +0 -0
  83. alp_data-1.10.0/docs/backends.md +278 -0
  84. alp_data-1.10.0/docs/chain.md +160 -0
  85. alp_data-1.10.0/docs/concatenate.md +365 -0
  86. alp_data-1.10.0/docs/datasets.md +355 -0
  87. alp_data-1.10.0/docs/hooks/dataset_info_hook.py +145 -0
  88. alp_data-1.10.0/docs/index.md +162 -0
  89. alp_data-1.10.0/docs/io.md +426 -0
  90. alp_data-1.10.0/docs/reference.md +7 -0
  91. alp_data-1.10.0/docs/stylesheets/extra.css +75 -0
  92. alp_data-1.10.0/docs/transforms.md +186 -0
  93. alp_data-1.10.0/jobs/array_jobs/benchmark_latency_array_1.sh +36 -0
  94. alp_data-1.10.0/jobs/array_jobs/benchmark_latency_array_16.sh +37 -0
  95. alp_data-1.10.0/jobs/array_jobs/benchmark_latency_array_2.sh +37 -0
  96. alp_data-1.10.0/jobs/array_jobs/benchmark_latency_array_4.sh +37 -0
  97. alp_data-1.10.0/jobs/array_jobs/benchmark_latency_array_8.sh +37 -0
  98. alp_data-1.10.0/jobs/array_jobs/save_and_plot.sh +24 -0
  99. alp_data-1.10.0/jobs/benchmark_beans.sh +23 -0
  100. alp_data-1.10.0/jobs/benchmark_latency_array_exp.sh +24 -0
  101. alp_data-1.10.0/jobs/benchmark_latency_config.sh +78 -0
  102. alp_data-1.10.0/jobs/benchmark_latency_default.sh +67 -0
  103. alp_data-1.10.0/jobs/benchmark_loading_time_config.sh +48 -0
  104. alp_data-1.10.0/jobs/benchmark_loading_time_default.sh +38 -0
  105. alp_data-1.10.0/jobs/multi_dataset_latency.sh +30 -0
  106. alp_data-1.10.0/jobs/multi_dataset_loading_time.sh +32 -0
  107. alp_data-1.10.0/jobs/run_benchmark.sh +41 -0
  108. alp_data-1.10.0/mkdocs.yml +84 -0
  109. alp_data-1.10.0/pyproject.toml +133 -0
  110. alp_data-1.10.0/requirements-dev.txt +126 -0
  111. alp_data-1.10.0/requirements.txt +263 -0
  112. alp_data-1.10.0/scripts/benchmarks/README.md +135 -0
  113. alp_data-1.10.0/scripts/benchmarks/benchmark_config.yaml +13 -0
  114. alp_data-1.10.0/scripts/benchmarks/benchmark_dataset.py +339 -0
  115. alp_data-1.10.0/scripts/benchmarks/benchmark_latency.py +457 -0
  116. alp_data-1.10.0/scripts/benchmarks/benchmark_utils.py +301 -0
  117. alp_data-1.10.0/scripts/benchmarks/generate_list.py +22 -0
  118. alp_data-1.10.0/scripts/benchmarks/job_array_exp_no_resampling_config.yaml +13 -0
  119. alp_data-1.10.0/scripts/benchmarks/job_array_exp_resampling_config.yaml +15 -0
  120. alp_data-1.10.0/scripts/benchmarks/loading_time.py +156 -0
  121. alp_data-1.10.0/scripts/benchmarks/merge_array_results.py +44 -0
  122. alp_data-1.10.0/scripts/benchmarks/plot_latency_bench.py +127 -0
  123. alp_data-1.10.0/scripts/benchmarks/plot_loading_bench.py +191 -0
  124. alp_data-1.10.0/scripts/clean_animalspeak_splits.py +116 -0
  125. alp_data-1.10.0/scripts/configs/resample_birdset_librosa.yaml +11 -0
  126. alp_data-1.10.0/scripts/data_preprocessing_scripts/anuraset_strong.py +281 -0
  127. alp_data-1.10.0/scripts/data_preprocessing_scripts/arctic_bird_sounds.py +267 -0
  128. alp_data-1.10.0/scripts/data_preprocessing_scripts/audioset_010.py +127 -0
  129. alp_data-1.10.0/scripts/data_preprocessing_scripts/beans_zero_export_as_flac.py +270 -0
  130. alp_data-1.10.0/scripts/data_preprocessing_scripts/benchmark_birdset_resample.py +121 -0
  131. alp_data-1.10.0/scripts/data_preprocessing_scripts/birdeep.py +200 -0
  132. alp_data-1.10.0/scripts/data_preprocessing_scripts/birdset_010.py +363 -0
  133. alp_data-1.10.0/scripts/data_preprocessing_scripts/birdset_resample.py +2132 -0
  134. alp_data-1.10.0/scripts/data_preprocessing_scripts/birdset_resample_job.sh +52 -0
  135. alp_data-1.10.0/scripts/data_preprocessing_scripts/cache_gbif_taxonomy_conversion.py +407 -0
  136. alp_data-1.10.0/scripts/data_preprocessing_scripts/format_corvid_wascher.py +226 -0
  137. alp_data-1.10.0/scripts/data_preprocessing_scripts/format_hawaiian_birds.py +195 -0
  138. alp_data-1.10.0/scripts/data_preprocessing_scripts/inat_download.py +277 -0
  139. alp_data-1.10.0/scripts/data_preprocessing_scripts/inat_m4a_to_wav.py +343 -0
  140. alp_data-1.10.0/scripts/data_preprocessing_scripts/inat_mp3_to_wav.py +397 -0
  141. alp_data-1.10.0/scripts/data_preprocessing_scripts/inat_resample.py +200 -0
  142. alp_data-1.10.0/scripts/data_preprocessing_scripts/macaques_coo_calls_010.py +72 -0
  143. alp_data-1.10.0/scripts/data_preprocessing_scripts/nocturnal_bird_migration.py +531 -0
  144. alp_data-1.10.0/scripts/data_preprocessing_scripts/orchive_010.py +160 -0
  145. alp_data-1.10.0/scripts/data_preprocessing_scripts/powdermill_taxonomy_and_verification.py +246 -0
  146. alp_data-1.10.0/scripts/data_preprocessing_scripts/subsegmentation.py +490 -0
  147. alp_data-1.10.0/scripts/data_preprocessing_scripts/taxonomy_v2_source_to_tsv.py +85 -0
  148. alp_data-1.10.0/scripts/data_preprocessing_scripts/wabad_taxonomy_and_verification.py +228 -0
  149. alp_data-1.10.0/scripts/data_preprocessing_scripts/wabad_taxonomy_and_verification_v2.py +275 -0
  150. alp_data-1.10.0/scripts/data_preprocessing_scripts/xeno_canto_annotated_jeantet_2023.py +293 -0
  151. alp_data-1.10.0/scripts/data_preprocessing_scripts/xeno_canto_resample.py +136 -0
  152. alp_data-1.10.0/scripts/data_preprocessing_scripts/zebra_finch_julie_elie_010.py +270 -0
  153. alp_data-1.10.0/scripts/dataset_prep_notebooks/animalspeak_eval.ipynb +302 -0
  154. alp_data-1.10.0/scripts/dataset_prep_notebooks/issue_83_fix_beans_and_birdset.ipynb +1183 -0
  155. alp_data-1.10.0/scripts/dataset_prep_notebooks/prep_animalspeak_2.1.ipynb +62 -0
  156. alp_data-1.10.0/scripts/dataset_prep_notebooks/prep_asa_berlin.ipynb +395 -0
  157. alp_data-1.10.0/scripts/dataset_prep_notebooks/prep_barkey_canyon_detection.ipynb +4002 -0
  158. alp_data-1.10.0/scripts/dataset_prep_notebooks/prep_barkley_canyon.ipynb +2511 -0
  159. alp_data-1.10.0/scripts/dataset_prep_notebooks/prep_beans.ipynb +3840 -0
  160. alp_data-1.10.0/scripts/dataset_prep_notebooks/prep_dolphin_whistles.ipynb +325 -0
  161. alp_data-1.10.0/scripts/dataset_prep_notebooks/prep_egyptian_fruit_bats.ipynb +370 -0
  162. alp_data-1.10.0/scripts/dataset_prep_notebooks/prep_geladas.ipynb +1085 -0
  163. alp_data-1.10.0/scripts/dataset_prep_notebooks/prep_gibbons.ipynb +1001 -0
  164. alp_data-1.10.0/scripts/dataset_prep_notebooks/prep_humbugdb.ipynb +43 -0
  165. alp_data-1.10.0/scripts/dataset_prep_notebooks/prep_inaturalist2025.ipynb +2183 -0
  166. alp_data-1.10.0/scripts/dataset_prep_notebooks/prep_insectset.ipynb +2998 -0
  167. alp_data-1.10.0/scripts/dataset_prep_notebooks/prep_macaques.ipynb +949 -0
  168. alp_data-1.10.0/scripts/dataset_prep_notebooks/prep_raincoast.ipynb +1464 -0
  169. alp_data-1.10.0/scripts/dataset_prep_notebooks/prep_xenocanto_2025.ipynb +3489 -0
  170. alp_data-1.10.0/scripts/resample_datasets.py +1017 -0
  171. alp_data-1.10.0/tests/io/test_paths.py +63 -0
  172. alp_data-1.10.0/tests/io/test_paths_core.py +582 -0
  173. alp_data-1.10.0/tests/samples/file1.txt +1 -0
  174. alp_data-1.10.0/tests/samples/my_custom_dataset_cfg.yml +19 -0
  175. alp_data-1.10.0/tests/samples/noise.wav +0 -0
  176. alp_data-1.10.0/tests/samples/stereo.wav +0 -0
  177. alp_data-1.10.0/tests/samples/test_chain_config.yml +12 -0
  178. alp_data-1.10.0/tests/samples/test_collection_config.yml +19 -0
  179. alp_data-1.10.0/tests/samples/test_concat_config.yml +19 -0
  180. alp_data-1.10.0/tests/samples/test_wrong_config.yml +70 -0
  181. alp_data-1.10.0/tests/test_backends.py +283 -0
  182. alp_data-1.10.0/tests/test_balanced_sample.py +440 -0
  183. alp_data-1.10.0/tests/test_chained.py +73 -0
  184. alp_data-1.10.0/tests/test_cloud_paths.py +552 -0
  185. alp_data-1.10.0/tests/test_concat.py +646 -0
  186. alp_data-1.10.0/tests/test_dataset.py +305 -0
  187. alp_data-1.10.0/tests/test_deduplicate.py +38 -0
  188. alp_data-1.10.0/tests/test_fileio_functional.py +150 -0
  189. alp_data-1.10.0/tests/test_filter.py +136 -0
  190. alp_data-1.10.0/tests/test_label_from_feature.py +204 -0
  191. alp_data-1.10.0/tests/test_long_tail_upsample.py +223 -0
  192. alp_data-1.10.0/tests/test_multilabel_from_features.py +227 -0
  193. alp_data-1.10.0/tests/test_read_audio.py +244 -0
  194. alp_data-1.10.0/tests/test_read_utils.py +134 -0
  195. alp_data-1.10.0/tests/test_select_columns.py +123 -0
  196. alp_data-1.10.0/tests/test_streaming.py +392 -0
  197. alp_data-1.10.0/tests/test_subsample.py +165 -0
  198. alp_data-1.10.0/tests/test_taxonomy.py +377 -0
  199. alp_data-1.10.0/tests/test_utils.py +110 -0
  200. alp_data-1.10.0/tests/unittests/datasets_tests/test_animal_sound_archive.py +253 -0
  201. alp_data-1.10.0/tests/unittests/datasets_tests/test_animal_speak.py +430 -0
  202. alp_data-1.10.0/tests/unittests/datasets_tests/test_anuraset_strong.py +211 -0
  203. alp_data-1.10.0/tests/unittests/datasets_tests/test_arctic_bird_sounds.py +194 -0
  204. alp_data-1.10.0/tests/unittests/datasets_tests/test_audioset.py +323 -0
  205. alp_data-1.10.0/tests/unittests/datasets_tests/test_audioset_strong.py +177 -0
  206. alp_data-1.10.0/tests/unittests/datasets_tests/test_beans.py +231 -0
  207. alp_data-1.10.0/tests/unittests/datasets_tests/test_beans_zero.py +190 -0
  208. alp_data-1.10.0/tests/unittests/datasets_tests/test_bengalese_finch_calls.py +281 -0
  209. alp_data-1.10.0/tests/unittests/datasets_tests/test_birdeep.py +194 -0
  210. alp_data-1.10.0/tests/unittests/datasets_tests/test_birdset.py +225 -0
  211. alp_data-1.10.0/tests/unittests/datasets_tests/test_chiffchaff_id.py +164 -0
  212. alp_data-1.10.0/tests/unittests/datasets_tests/test_corvid_wascher.py +187 -0
  213. alp_data-1.10.0/tests/unittests/datasets_tests/test_dclde2026.py +252 -0
  214. alp_data-1.10.0/tests/unittests/datasets_tests/test_dinardo_dolphin_whistles.py +166 -0
  215. alp_data-1.10.0/tests/unittests/datasets_tests/test_esp_raincoast.py +208 -0
  216. alp_data-1.10.0/tests/unittests/datasets_tests/test_geladas.py +173 -0
  217. alp_data-1.10.0/tests/unittests/datasets_tests/test_giant_otters.py +252 -0
  218. alp_data-1.10.0/tests/unittests/datasets_tests/test_gibbon_solos.py +173 -0
  219. alp_data-1.10.0/tests/unittests/datasets_tests/test_hawaiian_birds.py +203 -0
  220. alp_data-1.10.0/tests/unittests/datasets_tests/test_inaturalist.py +251 -0
  221. alp_data-1.10.0/tests/unittests/datasets_tests/test_infant_marmosets_vox.py +383 -0
  222. alp_data-1.10.0/tests/unittests/datasets_tests/test_insectset_459.py +199 -0
  223. alp_data-1.10.0/tests/unittests/datasets_tests/test_littleowl_id.py +72 -0
  224. alp_data-1.10.0/tests/unittests/datasets_tests/test_macaques_coo_calls.py +295 -0
  225. alp_data-1.10.0/tests/unittests/datasets_tests/test_nocturnal_bird_migration.py +203 -0
  226. alp_data-1.10.0/tests/unittests/datasets_tests/test_pipit_id.py +77 -0
  227. alp_data-1.10.0/tests/unittests/datasets_tests/test_powdermill.py +188 -0
  228. alp_data-1.10.0/tests/unittests/datasets_tests/test_subsegmentation.py +312 -0
  229. alp_data-1.10.0/tests/unittests/datasets_tests/test_superb_starling.py +240 -0
  230. alp_data-1.10.0/tests/unittests/datasets_tests/test_voxaboxen.py +166 -0
  231. alp_data-1.10.0/tests/unittests/datasets_tests/test_wabad.py +228 -0
  232. alp_data-1.10.0/tests/unittests/datasets_tests/test_watkins.py +275 -0
  233. alp_data-1.10.0/tests/unittests/datasets_tests/test_xeno_canto.py +304 -0
  234. alp_data-1.10.0/tests/unittests/datasets_tests/test_xeno_canto_annotated_jeantet_23.py +212 -0
  235. alp_data-1.10.0/tests/unittests/datasets_tests/test_zebra_finch_julie_elie.py +261 -0
  236. alp_data-1.10.0/uv.lock +3701 -0
@@ -0,0 +1,10 @@
1
+ ## Contents ##
2
+
3
+ ### Common Words for use in Compound Words ###
4
+ ### Compound Words With 1 or 2 letter Words ###
5
+ ### Jargon ###
6
+ Coo
7
+ coo
8
+ ### Names ###
9
+ ### British ###
10
+ ### Non-English ###
@@ -0,0 +1 @@
1
+ * @earthspecies/infra-admins
@@ -0,0 +1,66 @@
1
+ name: Tests CI
2
+
3
+ on: # yamllint disable-line rule:truthy
4
+ workflow_dispatch:
5
+ push:
6
+ branches: ["main"]
7
+ pull_request:
8
+
9
+ jobs:
10
+ tests:
11
+ name: Run pytests tests
12
+ runs-on: self-hosted
13
+
14
+ steps:
15
+ - uses: actions/checkout@v4
16
+
17
+ - name: Install uv
18
+ uses: astral-sh/setup-uv@v4
19
+ with:
20
+ enable-cache: "auto"
21
+
22
+ - name: "Set up Python"
23
+ uses: actions/setup-python@v5
24
+ with:
25
+ python-version-file: ".python-version"
26
+
27
+ - name: Install the project
28
+ run: uv sync --all-extras --dev
29
+
30
+ - name: Display Python version
31
+ run: python -c "import sys; print(sys.version)"
32
+
33
+ - name: Checking dependencies
34
+ run: uv run deptry .
35
+
36
+ - name: Run Doctests
37
+ run: uv run pytest --doctest-modules alp_data
38
+
39
+ - name: Run tests
40
+ run: uv run pytest tests
41
+
42
+ test-py-versions-self-hosted:
43
+ name: Run tests on multiple Python versions
44
+ runs-on: self-hosted
45
+ strategy:
46
+ matrix:
47
+ python-version: ["3.11", "3.12", "3.13", "3.14"]
48
+
49
+ steps:
50
+ - uses: actions/checkout@v4
51
+
52
+ - name: Install uv
53
+ uses: astral-sh/setup-uv@v4
54
+ with:
55
+ enable-cache: true
56
+
57
+ - name: "Set up Python ${{ matrix.python-version }}"
58
+ uses: actions/setup-python@v5
59
+ with:
60
+ python-version: ${{ matrix.python-version }}
61
+
62
+ - name: Install the project
63
+ run: uv sync --all-extras --dev
64
+
65
+ - name: Run core tests
66
+ run: uv run pytest tests/io
@@ -0,0 +1,58 @@
1
+ name: Deploy docs
2
+
3
+ on:
4
+ workflow_dispatch:
5
+ push:
6
+ branches:
7
+ - main
8
+ tags:
9
+ - v[0-9]+.[0-9]+*
10
+ - latest
11
+
12
+ concurrency:
13
+ group: docs-deploy
14
+ cancel-in-progress: false
15
+
16
+ jobs:
17
+ deploy-docs:
18
+ runs-on: ubuntu-latest
19
+ permissions:
20
+ contents: write
21
+
22
+ steps:
23
+ - uses: actions/checkout@v4
24
+ with:
25
+ fetch-depth: 0
26
+
27
+ - name: Install uv
28
+ uses: astral-sh/setup-uv@v4
29
+ with:
30
+ enable-cache: true
31
+
32
+ - name: "Set up Python"
33
+ uses: actions/setup-python@v5
34
+ with:
35
+ python-version-file: ".python-version"
36
+
37
+ - name: Install the project
38
+ run: uv sync --all-extras --dev
39
+
40
+ - name: Configure Git user
41
+ run: |
42
+ git config --local user.name "${{ github.actor }}"
43
+ git config --local user.email "${{ github.actor }}@users.noreply.github.com"
44
+
45
+ - name: Deploy main branch
46
+ if: github.ref == 'refs/heads/main'
47
+ run: uv run mike deploy -b docs-site --push --update-aliases dev
48
+
49
+ - name: Deploy version from tag
50
+ if: startsWith(github.ref, 'refs/tags/v')
51
+ run: |
52
+ VERSION=${GITHUB_REF#refs/tags/v}
53
+ uv run mike deploy -b docs-site --push --update-aliases $VERSION
54
+
55
+ - name: Update latest tag
56
+ if: github.ref == 'refs/tags/latest'
57
+ run: |
58
+ uv run mike deploy -b docs-site --push --update-aliases latest
@@ -0,0 +1,30 @@
1
+ name: Validate docs
2
+
3
+ on:
4
+ pull_request:
5
+
6
+ jobs:
7
+ validate-docs:
8
+ runs-on: ubuntu-latest
9
+ concurrency:
10
+ group: docs-validate-${{ github.ref }}
11
+ cancel-in-progress: true
12
+
13
+ steps:
14
+ - uses: actions/checkout@v4
15
+
16
+ - name: Install uv
17
+ uses: astral-sh/setup-uv@v4
18
+ with:
19
+ enable-cache: true
20
+
21
+ - name: "Set up Python"
22
+ uses: actions/setup-python@v5
23
+ with:
24
+ python-version-file: ".python-version"
25
+
26
+ - name: Install the project
27
+ run: uv sync --all-extras --dev
28
+
29
+ - name: Build docs (dry run)
30
+ run: uv run mkdocs build --strict
@@ -0,0 +1,16 @@
1
+ name: pre-commit hooks
2
+
3
+ on: # yamllint disable-line rule:truthy
4
+ push:
5
+ branches: [main]
6
+ pull_request:
7
+
8
+ jobs:
9
+ pre-commit:
10
+ runs-on: ubuntu-latest
11
+ steps:
12
+ - uses: actions/checkout@v4
13
+ - uses: actions/setup-python@v5
14
+ with:
15
+ python-version-file: ".python-version"
16
+ - uses: j178/prek-action@v2
@@ -0,0 +1,35 @@
1
+ name: Publish Package to PyPI
2
+
3
+ on: # yamllint disable-line rule:truthy
4
+ workflow_dispatch:
5
+ release:
6
+ types: [published]
7
+
8
+ jobs:
9
+ publish:
10
+ runs-on: ubuntu-latest
11
+ permissions:
12
+ contents: read
13
+ id-token: write
14
+
15
+ steps:
16
+ - uses: actions/checkout@v4
17
+
18
+ - name: Install uv
19
+ uses: astral-sh/setup-uv@v4
20
+ with:
21
+ enable-cache: true
22
+
23
+ - name: Set up Python
24
+ uses: actions/setup-python@v5
25
+ with:
26
+ python-version-file: ".python-version"
27
+
28
+ - name: Install the project
29
+ run: uv sync --all-extras --dev
30
+
31
+ - name: Build the package
32
+ run: uv build
33
+
34
+ - name: Publish to PyPI
35
+ run: uv publish --publish-url https://upload.pypi.org/legacy/ --trusted-publishing always --keyring-provider disabled
@@ -0,0 +1,184 @@
1
+ # data
2
+ tmp/
3
+
4
+ # Byte-compiled / optimized / DLL files
5
+ __pycache__/
6
+ *.py[cod]
7
+ *$py.class
8
+
9
+ # C extensions
10
+ *.so
11
+
12
+ # Distribution / packaging
13
+ .Python
14
+ build/
15
+ develop-eggs/
16
+ dist/
17
+ downloads/
18
+ eggs/
19
+ .eggs/
20
+ lib/
21
+ lib64/
22
+ parts/
23
+ sdist/
24
+ var/
25
+ wheels/
26
+ share/python-wheels/
27
+ *.egg-info/
28
+ .installed.cfg
29
+ *.egg
30
+ MANIFEST
31
+
32
+ # PyInstaller
33
+ # Usually these files are written by a python script from a template
34
+ # before PyInstaller builds the exe, so as to inject date/other infos into it.
35
+ *.manifest
36
+ *.spec
37
+
38
+ # Installer logs
39
+ pip-log.txt
40
+ pip-delete-this-directory.txt
41
+
42
+ # Unit test / coverage reports
43
+ htmlcov/
44
+ .tox/
45
+ .nox/
46
+ .coverage
47
+ .coverage.*
48
+ .cache
49
+ .ruff_cache
50
+ nosetests.xml
51
+ coverage.xml
52
+ *.cover
53
+ *.py,cover
54
+ .hypothesis/
55
+ .pytest_cache/
56
+ cover/
57
+ .ipynb_checkpoints
58
+
59
+ # AI
60
+ .claude
61
+
62
+ # Translations
63
+ *.mo
64
+ *.pot
65
+
66
+ # Django stuff:
67
+ *.log
68
+ local_settings.py
69
+ db.sqlite3
70
+ db.sqlite3-journal
71
+
72
+ # Flask stuff:
73
+ instance/
74
+ .webassets-cache
75
+
76
+ # Scrapy stuff:
77
+ .scrapy
78
+
79
+ # Sphinx documentation
80
+ docs/_build/
81
+
82
+ # PyBuilder
83
+ .pybuilder/
84
+ target/
85
+
86
+ # Jupyter Notebook
87
+ .ipynb_checkpoints
88
+
89
+ # IPython
90
+ profile_default/
91
+ ipython_config.py
92
+
93
+ # pyenv
94
+ # For a library or package, you might want to ignore these files since the code is
95
+ # intended to run in multiple environments; otherwise, check them in:
96
+ # .python-version
97
+
98
+ # pipenv
99
+ # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
100
+ # However, in case of collaboration, if having platform-specific dependencies or dependencies
101
+ # having no cross-platform support, pipenv may install dependencies that don't work, or not
102
+ # install all needed dependencies.
103
+ #Pipfile.lock
104
+
105
+ # UV
106
+ # Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
107
+ # This is especially recommended for binary packages to ensure reproducibility, and is more
108
+ # commonly ignored for libraries.
109
+ #uv.lock
110
+
111
+ # poetry
112
+ # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
113
+ # This is especially recommended for binary packages to ensure reproducibility, and is more
114
+ # commonly ignored for libraries.
115
+ # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
116
+ #poetry.lock
117
+
118
+ # pdm
119
+ # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
120
+ #pdm.lock
121
+ # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
122
+ # in version control.
123
+ # https://pdm.fming.dev/latest/usage/project/#working-with-version-control
124
+ .pdm.toml
125
+ .pdm-python
126
+ .pdm-build/
127
+
128
+ # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
129
+ __pypackages__/
130
+
131
+ # Celery stuff
132
+ celerybeat-schedule
133
+ celerybeat.pid
134
+
135
+ # SageMath parsed files
136
+ *.sage.py
137
+
138
+ # Environments
139
+ .env
140
+ .venv
141
+ env/
142
+ venv/
143
+ ENV/
144
+ env.bak/
145
+ venv.bak/
146
+
147
+ # Spyder project settings
148
+ .spyderproject
149
+ .spyproject
150
+
151
+ # Rope project settings
152
+ .ropeproject
153
+
154
+ # mkdocs documentation
155
+ /site
156
+
157
+ # mypy
158
+ .mypy_cache/
159
+ .dmypy.json
160
+ dmypy.json
161
+
162
+ # Pyre type checker
163
+ .pyre/
164
+
165
+ # pytype static type analyzer
166
+ .pytype/
167
+
168
+ # Cython debug symbols
169
+ cython_debug/
170
+
171
+ # PyCharm
172
+ # JetBrains specific template is maintained in a separate JetBrains.gitignore that can
173
+ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
174
+ # and can be added to the global gitignore or merged into this file. For a more nuclear
175
+ # option (not recommended) you can uncomment the following to ignore the entire idea folder.
176
+ #.idea/
177
+
178
+ # PyPI configuration file
179
+ .pypirc
180
+ .aider*
181
+
182
+ # Benchmark needs
183
+ scripts/benchmarks/fig/
184
+ scripts/benchmarks/dataset_list.txt
@@ -0,0 +1,52 @@
1
+ repos:
2
+ - repo: https://github.com/pre-commit/pre-commit-hooks
3
+ rev: v2.5.0
4
+ hooks:
5
+ - id: trailing-whitespace
6
+ types: [file, text]
7
+ - id: end-of-file-fixer
8
+ types: [python]
9
+ - id: mixed-line-ending
10
+ types: [python]
11
+ args: [--fix=no]
12
+ - id: check-added-large-files
13
+ args: [--maxkb=1024]
14
+
15
+ - repo: https://github.com/astral-sh/ruff-pre-commit
16
+ rev: v0.15.10
17
+ hooks:
18
+ - id: ruff
19
+ args: [--fix]
20
+ - id: ruff-format
21
+
22
+ - repo: https://github.com/adrienverge/yamllint
23
+ rev: v1.35.1
24
+ hooks:
25
+ - id: yamllint
26
+
27
+ - repo: https://github.com/codespell-project/codespell
28
+ rev: v2.3.0
29
+ hooks:
30
+ - id: codespell
31
+ args:
32
+ - "--ignore-words=.dict-allowed.txt"
33
+ # skip jupyter notebook as there isn't a good way to only match inputs
34
+ # at the moment. manually fixing up outputs would be a pain and we
35
+ # cannot always expect to regex them out.
36
+ - "--ignore-regex='base64,.*?=='"
37
+ additional_dependencies:
38
+ - tomli
39
+
40
+ - repo: https://github.com/astral-sh/uv-pre-commit
41
+ rev: 0.11.2
42
+ hooks:
43
+ - id: uv-lock
44
+ files: ^(uv\.lock|pyproject\.toml|uv\.toml)$
45
+ - id: uv-export
46
+ # yamllint disable-line rule:line-length
47
+ args: ["--frozen", "--output-file=requirements.txt", "--no-dev", "--no-hashes"]
48
+ files: ^(uv\.lock|pyproject\.toml|uv\.toml)$
49
+ - id: uv-export
50
+ # yamllint disable-line rule:line-length
51
+ args: ["--frozen", "--output-file=requirements-dev.txt", "--only-dev", "--no-hashes"]
52
+ files: ^(uv\.lock|pyproject\.toml|uv\.toml)$
@@ -0,0 +1 @@
1
+ 3.11
@@ -0,0 +1,6 @@
1
+ extends: default
2
+
3
+ rules:
4
+ line-length: disable
5
+ comments-indentation: disable
6
+ document-start: disable
@@ -0,0 +1,14 @@
1
+ # Code style
2
+ - Use NumPy style for docstrings. If any exceptions are raised make sure that they're documented in the "Raises" section.
3
+ - When you make changes to the logic of a function or class make sure its docstring is still valid. If required, update the docstrong to match the changes.
4
+ - Run `ruff check --fix` to make sure your code is formatter correctly.
5
+ - Use single backticks for inline code references in docstrings
6
+ - Don't use conditional `if TYPE_CHECKING`
7
+ - Always add a `__all__` in `__init__.py` with strings of units that are exported
8
+ - Make sure that only objects that are needed outside a module are exposed in the `__init__.py`. Tests don't count as external usage; they should import directly from the submodule if something isn't in `__init__.py`.
9
+
10
+ # Key Directories
11
+ - `alp_data` - main python library
12
+
13
+ # Environment Setup
14
+ - Make sure you use `uv run` for running python commands
@@ -0,0 +1 @@
1
+ @AGENTS.md
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Earth Species Project
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,8 @@
1
+ .PHONY: serve-remote-docs serve-local-docs
2
+
3
+ serve-remote-docs:
4
+ git fetch origin
5
+ uv run mike serve -b origin/docs-site
6
+
7
+ serve-local-docs:
8
+ uv run mkdocs serve
@@ -0,0 +1,159 @@
1
+ Metadata-Version: 2.4
2
+ Name: alp-data
3
+ Version: 1.10.0
4
+ Summary: Data tools for ESP projects
5
+ License-File: LICENSE
6
+ Requires-Python: >=3.11
7
+ Requires-Dist: gcsfs>=2024.9.0.post1
8
+ Requires-Dist: google-cloud-secret-manager>=2.23.1
9
+ Requires-Dist: google-cloud-storage>=2.19.0
10
+ Requires-Dist: intervaltree>=3.1.0
11
+ Requires-Dist: librosa>=0.11.0
12
+ Requires-Dist: pandas
13
+ Requires-Dist: polars[pyarrow]>=1.33.1
14
+ Requires-Dist: pydantic>=2
15
+ Requires-Dist: pyyaml>=6.0.2
16
+ Requires-Dist: resampy
17
+ Requires-Dist: s3fs>=2024.9.0
18
+ Requires-Dist: semver
19
+ Requires-Dist: soundfile>=0.13.1
20
+ Description-Content-Type: text/markdown
21
+
22
+ # alp-data
23
+
24
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
25
+
26
+ **alp-data** gives you unified access to dozens of bioacoustic datasets, including recordings from birds, marine mammals, primates, insects, anurans, and multi-taxon benchmarks. Every built-in dataset shares a common `Dataset` interface, with streaming, configurable transforms, and consistent loading regardless of source format.
27
+
28
+ ## Why alp-data
29
+
30
+ Bioacoustic recordings live in many places: Zenodo, OSF, GBIF, institutional repositories. Each dataset arrives with its own format, manifest schema, audio organization, sampling rate, and licensing posture. Researchers wanting to listen across datasets first have to write a custom loader per dataset, then a custom mixer for combining them.
31
+
32
+ alp-data removes that scaffolding. Every built-in dataset surfaces the same interface (`for sample in ds`, `ds[i]`, `len(ds)`), with `audio` and `sample_rate` keys returned for every sample. Dataset-specific keys carry labels, annotations, and other metadata. Sample-rate harmonization, label derivation, and split-aware loading are wired in.
33
+
34
+ You can stand up a multi-dataset benchmark, compare models across taxa, or stream species-specific audio for transfer learning without first becoming a data-engineering specialist.
35
+
36
+ ## What's included
37
+
38
+ alp-data ships with 30+ built-in datasets across:
39
+
40
+ - **Birds** — large benchmarks including BirdSet (6,800+ training hours, 10,000 species) and WABAD (1,192 species, 72 sites); aggregator corpora like Xeno-Canto; and site- or species-specific recordings spanning arctic species, Hawaiian soundscapes, the Powdermill dawn chorus, and individual-ID datasets for chiffchaff, little owl, and tree pipit.
41
+ - **Marine mammals** — the Watkins Marine Mammal Sound Database (~13,700 clips across ~50 cetacean and pinniped species), DCLDE 2026 killer whale annotations, dolphin whistle and click corpora
42
+ - **Primates** — gelada vocal sequences, gibbon solos, infant marmoset vocalizations, and macaque coo calls.
43
+ - **Insects, anurans, and other mammals** — InsectSet459 (459 Orthoptera and Cicadidae species), AnuraSetStrong (42 frog species, 27 hours of expert annotations), and giant otter vocalization types.
44
+ - **Multi-taxon benchmarks and aggregators** — BEANS and BeansZero (the canonical bioacoustic and zero-shot benchmarks), AnimalSpeak (1M+ audio–caption pairs), AnimalSoundArchive, iNaturalist audio, AudioSet / AudioSetStrong, and Voxaboxen (overlapping vocalization detection).
45
+
46
+ Most datasets are openly licensed (CC-BY, CC-BY-NC, CC0, public domain). License and source metadata are available per-dataset via `Dataset.info` — provenance matters.
47
+
48
+ ## Quickstart
49
+
50
+ ```python
51
+ from alp_data import Beans
52
+
53
+ # Load 'train' split of the BEANS bioacoustic benchmark at 16kHz.
54
+ # Resampling is done on the fly via librosa.
55
+ beans = Beans(split="train", sample_rate=16000)
56
+
57
+ print(len(beans))
58
+
59
+ # Iterate
60
+ for sample in beans:
61
+ print(sample["audio"].shape)
62
+ break
63
+
64
+ # Indexed access
65
+ sample = beans[0]
66
+ print(sample["audio"].shape)
67
+
68
+ # Streaming mode (lower memory; len() not available)
69
+ beans_streaming = Beans(split="train", streaming=True)
70
+ for sample in beans_streaming:
71
+ print(sample["audio"].shape)
72
+ break
73
+ ```
74
+
75
+ > ⚠️ **Warning:** When using a PyTorch `DataLoader` with `num_workers > 0`, you must set the multiprocessing start method to `"spawn"` (not the default `"fork"` on Linux). alp-data datasets hold cloud-backed I/O handles (fsspec / `gcsfs` / `s3fs`) that are not safe to inherit across a `fork`; using `"fork"` can deadlock workers or corrupt audio reads. Either call `torch.multiprocessing.set_start_method("spawn", force=True)` at the top of your program, or pass a `"spawn"` context to the DataLoader, e.g.:
76
+ >
77
+ > ```python
78
+ > import torch.multiprocessing as mp
79
+ > from torch.utils.data import DataLoader
80
+ >
81
+ > loader = DataLoader(
82
+ > dataset,
83
+ > num_workers=4,
84
+ > multiprocessing_context=mp.get_context("spawn"),
85
+ > )
86
+ > ```
87
+
88
+ Datasets and transforms can also be loaded from a YAML config:
89
+
90
+ ```yaml
91
+ # config.yaml
92
+ dataset:
93
+ dataset_name: beans
94
+ split: train
95
+ sample_rate: 16000
96
+ transformations:
97
+ - type: filter
98
+ property: source_dataset
99
+ mode: include
100
+ values: ["watkins"]
101
+ ```
102
+
103
+ ```python
104
+ from alp_data import dataset_from_config
105
+
106
+ ds, transform_metadata = dataset_from_config("config.yaml")
107
+
108
+ # ds now only contains "watkins" samples from the BEANS train split, resampled to 16kHz.
109
+ ```
110
+
111
+ ## Installation
112
+ ```sh
113
+ git clone https://github.com/earthspecies/alp-data.git
114
+ cd alp-data
115
+ pip install -e . # or uv sync
116
+ ```
117
+
118
+ ## Development
119
+
120
+ This repository uses [uv](https://docs.astral.sh/uv/) for dependency management.
121
+
122
+ ```sh
123
+ # Install all dependencies including dev tools
124
+ uv sync --dev
125
+
126
+ # Set up pre-commit hooks (uses prek; pass --overwrite to replace existing pre-commit lib hooks)
127
+ uv run prek install
128
+
129
+ # Run tests (Note: some tests require authentication via google or cloudflare; see test files for details)
130
+ uv run pytest
131
+ ```
132
+
133
+ ## Documentation
134
+ Build documentation locally with:
135
+
136
+ ```sh
137
+ make serve-local-docs
138
+ # Hosts docs at http://localhost:8000
139
+ ```
140
+
141
+ ## Highlights
142
+
143
+ - **Source-flexible loading** — built-in datasets stream from a public Cloudflare R2 bucket; the I/O module can also read from local paths, GCS, S3, and other R2 buckets, with CSV / JSON-Lines / Parquet manifests supported at the backend layer.
144
+ - **Iterate or random-access indexing** — every Dataset supports `for sample in ds` and `ds[i]`.
145
+ - **Streaming mode** — process datasets larger than memory with `streaming=True`.
146
+ - **Configurable transforms** — filter rows, select columns, derive labels, deduplicate, balance, upsample long tails, subsample by ratio.
147
+ - **Combine datasets** — `ConcatenatedDataset` merges multiple datasets with configurable column-merge strategies; `ChainedDataset` iterates over them sequentially with streaming support.
148
+ - **Pluggable backends** — pandas or polars under the hood, selectable per-Dataset.
149
+
150
+
151
+ ## License
152
+
153
+ alp-data is released under the MIT License. See [LICENSE](LICENSE) for the full text.
154
+
155
+ The datasets accessed through alp-data are governed by their own licenses, set by their original creators — independent of alp-data's code license. Most are openly licensed (CC-BY, CC-BY-NC, CC0, public domain), but terms vary per dataset and may include attribution, share-alike, or non-commercial restrictions. Per-dataset license and source metadata are available via `Dataset.info`.
156
+
157
+ ## Contributing
158
+
159
+ For improvements, bug reports, or proposals, open an issue or pull request.