a3em-analysis 0.0.1__tar.gz

a3em_analysis-0.0.1/.gitignore

@@ -0,0 +1,6 @@
+.venv
+.env
+__pycache__
+.DS_Store
+src/a3em/.devenv
+dist/

a3em_analysis-0.0.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) Vanderbilt University, A3EM Project Team
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

a3em_analysis-0.0.1/PKG-INFO

@@ -0,0 +1,410 @@
+Metadata-Version: 2.4
+Name: a3em-analysis
+Version: 0.0.1
+Summary: A3EM provides utilities for preprocessing bioacoustic recordings, extracting acoustic features, and working with supported bioacoustic datasets.
+Project-URL: Homepage, https://github.com/vu-a3em/a3em-python-package
+Project-URL: Issues, https://github.com/vu-a3em/a3em-python-package/issues
+Author-email: Gabriel Barnard <gabriel.h.barnard@vanderbilt.edu>
+License-Expression: MIT
+License-File: LICENSE
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+
+# A3EM Package
+
+```python
+import a3em
+```
+
+A3EM provides utilities for preprocessing bioacoustic recordings, extracting acoustic features, and working with supported bioacoustic datasets.
+
+# `a3em.utils`
+
+## `preprocess`
+
+```python
+preprocess(audio, sample_rate, normalization=0.7)
+```
+
+Takes an audio clip as a `numpy.ndarray`, applies high-pass and low-pass filtering, and normalizes the signal.
+
+### Example
+
+```python
+import librosa
+from a3em.utils import preprocess
+
+audio_path = "test.wav"
+audio, sample_rate = librosa.load(audio_path)
+
+preprocessed_audio = preprocess(audio, sample_rate)
+```
+
+---
+
+## `extract_features`
+
+```python
+extract_features(audio, sample_rate)
+```
+
+Takes an audio clip as a `numpy.ndarray` and extracts acoustic features from the signal.
+
+### Example
+
+```python
+import librosa
+from a3em.utils import preprocess, extract_features
+
+audio_path = "test.wav"
+audio, sample_rate = librosa.load(audio_path)
+
+preprocessed_audio = preprocess(audio, sample_rate)
+features = extract_features(preprocessed_audio, sample_rate)
+```
+
+# `a3em.datasets`
+
+The `a3em.datasets` module provides convenient access to supported A3EM datasets. Dataset classes handle downloading, preprocessing, clip extraction, and feature extraction.
+
+## Arden
+
+The `Arden` dataset contains collar-borne AudioMoth recordings collected in June 2025 in Samburu National Reserve, Kenya.
+
+```python
+from a3em.datasets import Arden
+import os
+
+dataset = Arden(
+    path=os.getenv("DATA_PATH"),
+    token=os.getenv("API_TOKEN")
+)
+```
+
+`path` specifies where the dataset should be stored locally. `token` is used to authenticate downloads from Dryad.
+
+The first time data is loaded, the dataset will automatically:
+
+1. Check for the required audio and annotation files locally.
+2. Download the dataset from Dryad if necessary.
+3. Extract downloaded archives.
+4. Load and filter annotation metadata.
+5. Optionally generate background-noise examples.
+6. Extract audio clips.
+7. Preprocess the clips.
+8. Compute acoustic features.
+
+The extracted clips and features are cached on the `Arden` instance so that subsequent calls do not repeat feature extraction unless `reload=True` is specified.
+
+---
+
+## `load_data`
+
+```python
+load_data(
+    random_state=None,
+    sample_rate=2000,
+    rumble_only=False,
+    reload=False
+)
+```
+
+Loads the dataset and returns the extracted acoustic features together with their binary labels.
+
+### Example
+
+```python
+features, labels = dataset.load_data(
+    random_state=123
+)
+```
+
+### Parameters
+
+| Parameter      | Description                                                                                                        |
+| -------------- | ------------------------------------------------------------------------------------------------------------------ |
+| `random_state` | Seed used when shuffling metadata and generating background-noise clips.                                           |
+| `sample_rate`  | Sample rate used when loading audio. Defaults to `2000`.                                                           |
+| `rumble_only`  | If `True`, only annotated elephant rumbles are included. If `False`, background-noise examples are also generated. |
+| `reload`       | If `True`, regenerates metadata, clips, and features even if they have already been loaded.                        |
+
+### Returns
+
+```text
+features : pandas.DataFrame
+labels   : pandas.Series
+```
+
+Labels are binary:
+
+* `0` — Background noise
+* `1` — Elephant rumble
+
+Annotation quality values `2`, `3`, and `4` are mapped to the rumble label `1`. Background-noise examples have quality `0` and are mapped to label `0`.
+
+---
+
+## `load_data_ml`
+
+```python
+load_data_ml(
+    test_split=0.2,
+    random_state=None,
+    sample_rate=2000,
+    rumble_only=False,
+    reload=False,
+    shuffle=False
+)
+```
+
+Loads the dataset and creates train/test splits suitable for machine-learning workflows.
+
+Internally, this method uses `sklearn.model_selection.train_test_split`.
+
+### Example
+
+```python
+x_train, x_test, y_train, y_test = dataset.load_data_ml(
+    test_split=0.2,
+    random_state=123,
+    shuffle=True
+)
+```
+
+### Parameters
+
+| Parameter      | Description                                                                                                        |
+| -------------- | ------------------------------------------------------------------------------------------------------------------ |
+| `test_split`   | Fraction of samples reserved for testing. Defaults to `0.2`.                                                       |
+| `random_state` | Seed used for dataset generation and the train/test split.                                                         |
+| `sample_rate`  | Sample rate used when loading audio. Defaults to `2000`.                                                           |
+| `rumble_only`  | If `True`, only annotated elephant rumbles are included. If `False`, background-noise examples are also generated. |
+| `reload`       | If `True`, regenerates metadata, clips, and features before splitting.                                             |
+| `shuffle`      | Whether samples should be shuffled by `train_test_split` before creating the split. Defaults to `False`.           |
+
+### Returns
+
+```text
+x_train : pandas.DataFrame
+x_test  : pandas.DataFrame
+y_train : pandas.Series
+y_test  : pandas.Series
+```
+
+---
+
+## `load_clips`
+
+```python
+load_clips(
+    random_state=None,
+    sample_rate=2000,
+    rumble_only=False,
+    reload=False
+)
+```
+
+Returns the extracted audio clips together with their computed acoustic features.
+
+### Example
+
+```python
+clips, features = dataset.load_clips(
+    random_state=123,
+    rumble_only=False
+)
+```
+
+### Parameters
+
+| Parameter      | Description                                              |
+| -------------- | -------------------------------------------------------- |
+| `random_state` | Seed used when generating the dataset.                   |
+| `sample_rate`  | Sample rate used when loading audio. Defaults to `2000`. |
+| `rumble_only`  | If `True`, background-noise examples are not generated.  |
+| `reload`       | If `True`, regenerates the clips and features.           |
+
+### Returns
+
+* `clips` — list containing the extracted audio clips as NumPy arrays.
+* `features` — `pandas.DataFrame` containing one row of acoustic features for each clip.
+
+Each extracted clip includes a `0.2` second buffer before and after its annotated time range.
+
+Clips shorter than two seconds after extraction are discarded during feature extraction.
+
+---
+
+## Iteration
+
+An `Arden` dataset can be iterated over directly.
+
+```python
+dataset = Arden(path, token)
+
+for clip, features in dataset:
+    print(len(clip))
+    print(features)
+```
+
+If the dataset has not already been loaded, iteration automatically initializes it using the default loading options.
+
+Each iteration returns:
+
+```python
+(
+    numpy.ndarray,  # audio clip
+    dict            # extracted acoustic features
+)
+```
+
+---
+
+## Indexing
+
+Individual clips and their corresponding features can be accessed by index after the dataset has been loaded.
+
+```python
+clip, features = dataset[10]
+```
+
+The returned feature set is converted from its DataFrame row into a dictionary.
+
+If the dataset has not yet been loaded, indexing returns:
+
+```python
+(None, None)
+```
+
+---
+
+## Dataset Length
+
+The number of metadata entries currently loaded can be obtained with `len()`:
+
+```python
+len(dataset)
+```
+
+Before the dataset has been initialized, its length is `0`.
+
+---
+
+## Rumble Filtering
+
+Arden annotations are filtered before clips are extracted.
+
+Only entries satisfying all of the following conditions are retained:
+
+* `call_type` is `RUM` or `BKG`
+* `earflap` is `0` or `1`
+* `overlap` is `N`
+* `quality` is `0`, `2`, `3`, or `4`
+* duration is greater than `2` seconds
+
+When `rumble_only=True`, background-noise examples are not generated.
+
+---
+
+## Background-Noise Generation
+
+When `rumble_only=False`, background-noise (`BKG`) examples are automatically generated from regions outside the annotated event ranges.
+
+The duration of generated noise clips is based on the mean and standard deviation of annotation durations in the corresponding recording.
+
+Generated background-noise entries use:
+
+```text
+call_type = BKG
+quality   = 0
+overlap   = N
+earflap   = 0
+```
+
+Because background-noise selection is randomized, use `random_state` when reproducible dataset generation is required.
+
+```python
+features, labels = dataset.load_data(
+    random_state=123
+)
+```
+
+---
+
+## Audio Processing
+
+Audio recordings are loaded using `librosa` at the requested sample rate:
+
+```python
+sample_rate=2000
+```
+
+For each retained metadata entry:
+
+1. The corresponding time range is extracted from the recording.
+2. A `0.2` second buffer is added to each side.
+3. The clip is passed through `a3em.utils.preprocess`.
+4. Acoustic features are calculated using `a3em.utils.extract_features`.
+
+The original extracted clip and its computed features are retained by the dataset instance.
+
+---
+
+## Reloading Data
+
+Once clips and features have been generated, the `Arden` instance reuses them.
+
+To force the dataset to regenerate its metadata, clips, and features:
+
+```python
+features, labels = dataset.load_data(
+    reload=True
+)
+```
+
+This is useful when changing parameters such as:
+
+```python
+sample_rate
+rumble_only
+random_state
+```
+
+---
+
+## Quick Start
+
+```python
+import os
+from a3em.datasets import Arden
+
+dataset = Arden(
+    path=os.getenv("DATA_PATH"),
+    token=os.getenv("API_TOKEN")
+)
+
+# Load features and labels
+features, labels = dataset.load_data(
+    random_state=123
+)
+
+print(features.head())
+print(labels.head())
+
+# Create a machine-learning split
+x_train, x_test, y_train, y_test = dataset.load_data_ml(
+    test_split=0.2,
+    random_state=123,
+    shuffle=True
+)
+
+# Access raw clips and features
+clips, clip_features = dataset.load_clips()
+
+# Iterate through clips
+for clip, feature_set in dataset:
+    print(clip.shape)
+    print(feature_set)
+```

a3em_analysis-0.0.1/README.md

@@ -0,0 +1,396 @@
+# A3EM Package
+
+```python
+import a3em
+```
+
+A3EM provides utilities for preprocessing bioacoustic recordings, extracting acoustic features, and working with supported bioacoustic datasets.
+
+# `a3em.utils`
+
+## `preprocess`
+
+```python
+preprocess(audio, sample_rate, normalization=0.7)
+```
+
+Takes an audio clip as a `numpy.ndarray`, applies high-pass and low-pass filtering, and normalizes the signal.
+
+### Example
+
+```python
+import librosa
+from a3em.utils import preprocess
+
+audio_path = "test.wav"
+audio, sample_rate = librosa.load(audio_path)
+
+preprocessed_audio = preprocess(audio, sample_rate)
+```
+
+---
+
+## `extract_features`
+
+```python
+extract_features(audio, sample_rate)
+```
+
+Takes an audio clip as a `numpy.ndarray` and extracts acoustic features from the signal.
+
+### Example
+
+```python
+import librosa
+from a3em.utils import preprocess, extract_features
+
+audio_path = "test.wav"
+audio, sample_rate = librosa.load(audio_path)
+
+preprocessed_audio = preprocess(audio, sample_rate)
+features = extract_features(preprocessed_audio, sample_rate)
+```
+
+# `a3em.datasets`
+
+The `a3em.datasets` module provides convenient access to supported A3EM datasets. Dataset classes handle downloading, preprocessing, clip extraction, and feature extraction.
+
+## Arden
+
+The `Arden` dataset contains collar-borne AudioMoth recordings collected in June 2025 in Samburu National Reserve, Kenya.
+
+```python
+from a3em.datasets import Arden
+import os
+
+dataset = Arden(
+    path=os.getenv("DATA_PATH"),
+    token=os.getenv("API_TOKEN")
+)
+```
+
+`path` specifies where the dataset should be stored locally. `token` is used to authenticate downloads from Dryad.
+
+The first time data is loaded, the dataset will automatically:
+
+1. Check for the required audio and annotation files locally.
+2. Download the dataset from Dryad if necessary.
+3. Extract downloaded archives.
+4. Load and filter annotation metadata.
+5. Optionally generate background-noise examples.
+6. Extract audio clips.
+7. Preprocess the clips.
+8. Compute acoustic features.
+
+The extracted clips and features are cached on the `Arden` instance so that subsequent calls do not repeat feature extraction unless `reload=True` is specified.
+
+---
+
+## `load_data`
+
+```python
+load_data(
+    random_state=None,
+    sample_rate=2000,
+    rumble_only=False,
+    reload=False
+)
+```
+
+Loads the dataset and returns the extracted acoustic features together with their binary labels.
+
+### Example
+
+```python
+features, labels = dataset.load_data(
+    random_state=123
+)
+```
+
+### Parameters
+
+| Parameter      | Description                                                                                                        |
+| -------------- | ------------------------------------------------------------------------------------------------------------------ |
+| `random_state` | Seed used when shuffling metadata and generating background-noise clips.                                           |
+| `sample_rate`  | Sample rate used when loading audio. Defaults to `2000`.                                                           |
+| `rumble_only`  | If `True`, only annotated elephant rumbles are included. If `False`, background-noise examples are also generated. |
+| `reload`       | If `True`, regenerates metadata, clips, and features even if they have already been loaded.                        |
+
+### Returns
+
+```text
+features : pandas.DataFrame
+labels   : pandas.Series
+```
+
+Labels are binary:
+
+* `0` — Background noise
+* `1` — Elephant rumble
+
+Annotation quality values `2`, `3`, and `4` are mapped to the rumble label `1`. Background-noise examples have quality `0` and are mapped to label `0`.
+
+---
+
+## `load_data_ml`
+
+```python
+load_data_ml(
+    test_split=0.2,
+    random_state=None,
+    sample_rate=2000,
+    rumble_only=False,
+    reload=False,
+    shuffle=False
+)
+```
+
+Loads the dataset and creates train/test splits suitable for machine-learning workflows.
+
+Internally, this method uses `sklearn.model_selection.train_test_split`.
+
+### Example
+
+```python
+x_train, x_test, y_train, y_test = dataset.load_data_ml(
+    test_split=0.2,
+    random_state=123,
+    shuffle=True
+)
+```
+
+### Parameters
+
+| Parameter      | Description                                                                                                        |
+| -------------- | ------------------------------------------------------------------------------------------------------------------ |
+| `test_split`   | Fraction of samples reserved for testing. Defaults to `0.2`.                                                       |
+| `random_state` | Seed used for dataset generation and the train/test split.                                                         |
+| `sample_rate`  | Sample rate used when loading audio. Defaults to `2000`.                                                           |
+| `rumble_only`  | If `True`, only annotated elephant rumbles are included. If `False`, background-noise examples are also generated. |
+| `reload`       | If `True`, regenerates metadata, clips, and features before splitting.                                             |
+| `shuffle`      | Whether samples should be shuffled by `train_test_split` before creating the split. Defaults to `False`.           |
+
+### Returns
+
+```text
+x_train : pandas.DataFrame
+x_test  : pandas.DataFrame
+y_train : pandas.Series
+y_test  : pandas.Series
+```
+
+---
+
+## `load_clips`
+
+```python
+load_clips(
+    random_state=None,
+    sample_rate=2000,
+    rumble_only=False,
+    reload=False
+)
+```
+
+Returns the extracted audio clips together with their computed acoustic features.
+
+### Example
+
+```python
+clips, features = dataset.load_clips(
+    random_state=123,
+    rumble_only=False
+)
+```
+
+### Parameters
+
+| Parameter      | Description                                              |
+| -------------- | -------------------------------------------------------- |
+| `random_state` | Seed used when generating the dataset.                   |
+| `sample_rate`  | Sample rate used when loading audio. Defaults to `2000`. |
+| `rumble_only`  | If `True`, background-noise examples are not generated.  |
+| `reload`       | If `True`, regenerates the clips and features.           |
+
+### Returns
+
+* `clips` — list containing the extracted audio clips as NumPy arrays.
+* `features` — `pandas.DataFrame` containing one row of acoustic features for each clip.
+
+Each extracted clip includes a `0.2` second buffer before and after its annotated time range.
+
+Clips shorter than two seconds after extraction are discarded during feature extraction.
+
+---
+
+## Iteration
+
+An `Arden` dataset can be iterated over directly.
+
+```python
+dataset = Arden(path, token)
+
+for clip, features in dataset:
+    print(len(clip))
+    print(features)
+```
+
+If the dataset has not already been loaded, iteration automatically initializes it using the default loading options.
+
+Each iteration returns:
+
+```python
+(
+    numpy.ndarray,  # audio clip
+    dict            # extracted acoustic features
+)
+```
+
+---
+
+## Indexing
+
+Individual clips and their corresponding features can be accessed by index after the dataset has been loaded.
+
+```python
+clip, features = dataset[10]
+```
+
+The returned feature set is converted from its DataFrame row into a dictionary.
+
+If the dataset has not yet been loaded, indexing returns:
+
+```python
+(None, None)
+```
+
+---
+
+## Dataset Length
+
+The number of metadata entries currently loaded can be obtained with `len()`:
+
+```python
+len(dataset)
+```
+
+Before the dataset has been initialized, its length is `0`.
+
+---
+
+## Rumble Filtering
+
+Arden annotations are filtered before clips are extracted.
+
+Only entries satisfying all of the following conditions are retained:
+
+* `call_type` is `RUM` or `BKG`
+* `earflap` is `0` or `1`
+* `overlap` is `N`
+* `quality` is `0`, `2`, `3`, or `4`
+* duration is greater than `2` seconds
+
+When `rumble_only=True`, background-noise examples are not generated.
+
+---
+
+## Background-Noise Generation
+
+When `rumble_only=False`, background-noise (`BKG`) examples are automatically generated from regions outside the annotated event ranges.
+
+The duration of generated noise clips is based on the mean and standard deviation of annotation durations in the corresponding recording.
+
+Generated background-noise entries use:
+
+```text
+call_type = BKG
+quality   = 0
+overlap   = N
+earflap   = 0
+```
+
+Because background-noise selection is randomized, use `random_state` when reproducible dataset generation is required.
+
+```python
+features, labels = dataset.load_data(
+    random_state=123
+)
+```
+
+---
+
+## Audio Processing
+
+Audio recordings are loaded using `librosa` at the requested sample rate:
+
+```python
+sample_rate=2000
+```
+
+For each retained metadata entry:
+
+1. The corresponding time range is extracted from the recording.
+2. A `0.2` second buffer is added to each side.
+3. The clip is passed through `a3em.utils.preprocess`.
+4. Acoustic features are calculated using `a3em.utils.extract_features`.
+
+The original extracted clip and its computed features are retained by the dataset instance.
+
+---
+
+## Reloading Data
+
+Once clips and features have been generated, the `Arden` instance reuses them.
+
+To force the dataset to regenerate its metadata, clips, and features:
+
+```python
+features, labels = dataset.load_data(
+    reload=True
+)
+```
+
+This is useful when changing parameters such as:
+
+```python
+sample_rate
+rumble_only
+random_state
+```
+
+---
+
+## Quick Start
+
+```python
+import os
+from a3em.datasets import Arden
+
+dataset = Arden(
+    path=os.getenv("DATA_PATH"),
+    token=os.getenv("API_TOKEN")
+)
+
+# Load features and labels
+features, labels = dataset.load_data(
+    random_state=123
+)
+
+print(features.head())
+print(labels.head())
+
+# Create a machine-learning split
+x_train, x_test, y_train, y_test = dataset.load_data_ml(
+    test_split=0.2,
+    random_state=123,
+    shuffle=True
+)
+
+# Access raw clips and features
+clips, clip_features = dataset.load_clips()
+
+# Iterate through clips
+for clip, feature_set in dataset:
+    print(clip.shape)
+    print(feature_set)
+```

a3em_analysis-0.0.1/build.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+source .venv/bin/activate
+rm -rf dist/
+python -m build
+python -m twine upload dist/*
+deactivate

a3em_analysis-0.0.1/pyproject.toml

@@ -0,0 +1,34 @@
+[build-system]
+requires = [
+    "hatchling >= 1.26",
+    "pandas",
+    "numpy",
+    "librosa",
+    "soundfile",
+    "matplotlib",
+    "scikit-learn",
+    "python-dotenv",
+    "statsmodels",
+    "tqdm"
+]
+build-backend = "hatchling.build"
+
+[project]
+name = "a3em-analysis"
+version = "0.0.1"
+authors = [
+    { name="Gabriel Barnard", email="gabriel.h.barnard@vanderbilt.edu" },
+]
+description = "A3EM provides utilities for preprocessing bioacoustic recordings, extracting acoustic features, and working with supported bioacoustic datasets."
+readme = "README.md"
+requires-python = ">=3.8"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Operating System :: OS Independent",
+]
+license = "MIT"
+license-files = ["LICENSE"]
+
+[project.urls]
+Homepage = "https://github.com/vu-a3em/a3em-python-package"
+Issues = "https://github.com/vu-a3em/a3em-python-package/issues"

a3em_analysis-0.0.1/requirements.txt

@@ -0,0 +1,3 @@
+build
+twine
+setuptools

a3em_analysis-0.0.1/src/a3em/__init__.py

@@ -0,0 +1,2 @@
+from .utils import *
+from .datasets import *

a3em_analysis-0.0.1/src/a3em/datasets.py

@@ -0,0 +1,350 @@
+import a3em.utils
+import librosa
+import random
+import requests
+import zipfile
+import numpy as np
+import pandas as pd
+from abc import ABC, abstractmethod
+from pathlib import Path
+from sklearn.model_selection import train_test_split
+from tqdm import tqdm
+    
+class Dataset(ABC):
+
+    def __init__(self, path, token=None):
+        self.path = Path(path)
+        self.token = token
+        self._index = 0
+
+    @abstractmethod
+    def load_data(self, test_split, random_state, shuffle):
+        return None
+
+    @abstractmethod
+    def __iter__(self):
+        return None
+
+    @abstractmethod
+    def __next__(self):
+        return None
+
+    @abstractmethod
+    def __getitem__(self, key):
+        return None
+
+    @abstractmethod
+    def __len__(self):
+        return None
+ 
+class Arden(Dataset):
+
+    api = 'https://datadryad.org/'
+    doi = 'doi%3A10.5061%2Fdryad.xd2547dz3'
+
+    def __init__(self, path, token):
+        super().__init__(path, token)
+        self.prefetch = None
+        self.metadata = None
+        self.audiomoth_path = self.path / 'audiomoth'
+        self.annotations_path = self.path / 'manualAnnotations'
+        self._features = None
+        self._clips = None
+
+    def load_data(
+        self,
+        random_state=None,
+        sample_rate=2000,
+        rumble_only=False,
+        reload=False
+    ):
+        if self._clips is None or self._features is None or reload:
+            self.__setup(random_state, sample_rate, rumble_only)
+
+        labels = (
+            self.metadata['quality']
+            .replace({0: 0, 2: 1, 3: 1, 4: 1})
+            .rename('label')
+        )
+
+        return self._features, labels
+
+    def load_data_ml(
+        self, 
+        test_split=0.2, 
+        random_state=None, 
+        sample_rate=2000,
+        rumble_only=False, 
+        reload=False,
+        shuffle=False
+    ):        
+        if self._clips is None or self._features is None or reload:
+            self.__setup(random_state, sample_rate, rumble_only)
+
+        labels = (
+            self.metadata['quality']
+            .replace({0: 0, 2: 1, 3: 1, 4: 1})
+            .rename('label')
+        )
+
+        return train_test_split(
+            self._features,
+            labels,
+            test_size=test_split,
+            random_state=random_state,
+            shuffle=shuffle
+        )
+
+    def load_clips(
+        self, 
+        random_state=None, 
+        sample_rate=2000, 
+        rumble_only=False,
+        reload=False
+    ):
+        if self._clips is None or self._features is None or reload:
+            self.__setup(random_state, sample_rate, rumble_only)
+        return self._clips, self._features   
+
+    def __iter__(
+        self, 
+        random_state=None, 
+        sample_rate=2000, 
+        rumble_only=False,
+        reload=False
+    ):
+        if self._clips is None or self._features is None or reload:
+            self.__setup(random_state, sample_rate, rumble_only)
+        self._index = 0
+        return self
+            
+    def __next__(self):
+        if self._index == len(self):
+            raise StopIteration
+        clip = self._clips[self._index]
+        features = self._features.iloc[self._index].to_dict()
+        self._index += 1
+        return clip, features
+
+    def __getitem__(self, key):
+        if self._clips is None or self._features is None:
+            return None, None
+        return self._clips[key], self._features.iloc[key].to_dict()
+
+    def __len__(self):
+        return 0 if self.metadata is None else len(self.metadata)
+    
+    def __prefetch(self):
+        if not self.__validate_local_data():
+            self.__download_data()
+        audiomoth_files = sorted(self.audiomoth_path.glob('*.WAV'))
+        annotation_files = sorted(self.annotations_path.glob('*.txt'))
+        file_stems = [file.stem for file in audiomoth_files]
+        file_pairs = [
+            {'audio_path': x[0], 'annotation_path': x[1]}
+            for x in zip(audiomoth_files, annotation_files)
+        ]
+        self.prefetch = dict(zip(file_stems, file_pairs))      
+        print('prefetch complete')
+
+    def __setup(self, random_state, sample_rate, rumble_only):
+        self.__prefetch()
+        self.__load_metadata(random_state, rumble_only)
+        self.__load_audio_features(random_state, sample_rate)
+
+    def __validate_local_data(self):
+        if not (self.audiomoth_path.exists() and self.annotations_path.exists()):
+            return False
+        audiomoth_contents = sorted(self.audiomoth_path.glob('*.WAV'))
+        annotations_contents = sorted(self.annotations_path.glob('*.txt'))
+        annotations_stems = [file.stem for file in annotations_contents]
+        audiomoth_stems = [file.stem for file in audiomoth_contents]
+        return annotations_stems == audiomoth_stems
+
+    def __download_data(self):
+        # make sure the directory is clear
+        a3em.utils.clean_directory(self.path)
+
+        # pull files metadata
+        print('pulling metadata')
+        r = requests.get(f'{Arden.api}/api/v2/datasets/{Arden.doi}/versions')
+        if r.status_code != 200:
+            raise RuntimeError(r.text)
+        content = r.json()
+        latest_version = content['_embedded']['stash:versions'][0]
+        files_path = latest_version['_links']['stash:files']['href']
+
+        # get individual download links
+        print('locating files')
+        r = requests.get(f'{Arden.api}/{files_path}')
+        if r.status_code != 200:
+            raise RuntimeError(r.text)
+        content = r.json()
+        files = content['_embedded']['stash:files']
+
+        # download the files
+        print('download in progress')
+        for file in files:
+            download_src = file['_links']['stash:download']['href']
+            download_dst = self.path / file['path']
+            r = requests.get(f'{Arden.api}/{download_src}',
+                             headers={'authorization': f'Bearer {self.token}'})
+            if r.status_code != 200:
+                raise RuntimeError(r.text)
+            with open(download_dst, 'wb') as fd:
+                for chunk in r.iter_content(chunk_size=128):
+                    fd.write(chunk)
+
+        # unpack the files
+        print('unpacking')
+        zip_files = sorted(self.path.glob('*.zip'))
+        for zip_file in zip_files:
+            with zipfile.ZipFile(zip_file, 'r') as zip_ref:
+                zip_ref.extractall(self.path)
+            zip_file.unlink()
+
+    def __load_metadata(self, random_state, rumble_only):
+        metadata = pd.DataFrame()
+        for stem, prefetch in self.prefetch.items():
+            annotation_path = prefetch['annotation_path']
+            annotations = pd.read_csv(annotation_path, sep='\t')
+            if annotations.empty:
+                continue
+
+            # extract elephant rumbles
+            file_metadata = Arden.__filter_annotations(annotations, stem)
+
+            # extract background noise if applicable
+            if not rumble_only:
+                noise_metadata = Arden.__extract_noise(annotations, stem)
+                file_metadata = pd.concat(
+                    [file_metadata, noise_metadata], 
+                    ignore_index=True
+                )
+
+            # filter rumbles and add to dataframe
+            file_metadata = Arden.__filter_clips(file_metadata)
+            metadata = pd.concat([metadata, file_metadata], ignore_index=True)
+
+        self.metadata = (
+            metadata
+            .sample(frac=1, random_state=random_state)
+            .reset_index(drop=True)
+        )
+
+    def __load_audio_features(self, random_state=None, sample_rate=2000):
+        random.seed(random_state)
+        clips = [None] * len(self)
+        features = [None] * len(self)
+        
+        print('extracting features')
+        for stem, prefetch in tqdm(self.prefetch.items()):
+            # load in audio file
+            audio_file = prefetch['audio_path']
+            audio, _ = librosa.load(audio_file, sr=sample_rate)
+
+            df = self.metadata[self.metadata.file_stem == stem]
+            for index, row in df.iterrows():
+                start_time = row['Begin Time (s)']
+                end_time = row['End Time (s)']
+
+                clip, feature_set = Arden.__extract_clip_features(
+                    audio, 
+                    sample_rate, 
+                    start_time, 
+                    end_time
+                )
+
+                clips[index] = clip 
+                features[index] = feature_set
+        
+        self._features = pd.DataFrame(features)
+        self._clips = clips
+
+    @staticmethod
+    def __filter_annotations(annotations, stem):
+        d = ['Selection', 'View', 'Channel', 'Low Freq (Hz)', 'High Freq (Hz)']
+        df = annotations.drop(columns=d)
+        df.insert(0, 'file_stem', [stem] * len(annotations))
+        df['duration'] = df['End Time (s)'] - df['Begin Time (s)']
+        return df
+        
+    @staticmethod
+    def __extract_noise(rumble_annotations, stem):
+        # get ranges for all possible rumbles
+        rumble_event_ranges = list(zip(
+            rumble_annotations['Begin Time (s)'],
+            rumble_annotations['End Time (s)']
+        ))
+
+        # get average duration of rumbles
+        rumble_deltas = [end - start for start, end in rumble_event_ranges]
+        rumble_delta_stats = np.mean(rumble_deltas), np.std(rumble_deltas)
+
+        # generate noise clips from regions without rumbles
+        noise_regions = Arden.__find_noise_regions(rumble_event_ranges)
+        clip_ranges = [
+            Arden.__random_clip_range(region, rumble_delta_stats) 
+            for region in noise_regions
+        ]
+
+        # create dataframe
+        return pd.DataFrame([
+            {
+                'file_stem': stem,
+                'call_type': 'BKG',
+                'Begin Time (s)': clip_start,
+                'End Time (s)': clip_end,
+                'quality': 0,
+                'overlap': 'N',
+                'earflap': 0,
+                'duration': clip_end - clip_start
+            }
+            for clip_start, clip_end in clip_ranges
+        ])
+
+    @staticmethod
+    def __filter_clips(df):
+        df['earflap'] = pd.to_numeric(df['earflap'], errors='coerce')
+        return df[
+            (df['call_type'].isin(['RUM', 'BKG']))
+            & (df['earflap'].isin([0, 1]))
+            & (df['overlap'] == 'N')
+            & (df['quality'].isin([0, 2, 3, 4]))
+            & (df['duration'] > 2)
+        ]
+
+    @staticmethod
+    def __find_noise_regions(rumble_ranges):
+        start_times = [x[0] for x in rumble_ranges]
+        end_times = [x[1] for x in rumble_ranges]
+        start_times.pop()
+        start_times.insert(0, 0.0)
+        return list(zip(start_times, end_times))
+        
+    @staticmethod
+    def __random_clip_range(boundary, delta_stats):
+        start_bound, end_bound = boundary
+        delta_mean, delta_std = delta_stats
+        center = (end_bound - start_bound) * random.random() + start_bound
+        clip_length = (
+            delta_mean 
+            + delta_std 
+            * random.random() 
+            * random.randrange(-1, 2, 2)
+        )
+        clip_start = max(start_bound, center - clip_length / 2)
+        clip_end = min(center + clip_length / 2, end_bound)
+        return clip_start, clip_end
+
+    @staticmethod
+    def __extract_clip_features(audio, sample_rate, start_time, end_time):
+        buffer = 0.2
+        start_sample = max(0, int((start_time - buffer) * sample_rate))
+        end_sample = int((end_time + buffer) * sample_rate)
+        clip = audio[start_sample:end_sample]
+        if len(clip) < sample_rate * 2:
+            return [], {}
+        preprocessed_clip = a3em.utils.preprocess(clip, sample_rate)
+        features = a3em.utils.extract_features(preprocessed_clip, sample_rate)
+        return clip, features

a3em_analysis-0.0.1/src/a3em/requirements.txt

@@ -0,0 +1,10 @@
+pandas
+numpy
+librosa
+soundfile
+scipy
+matplotlib
+scikit-learn
+python-dotenv
+statsmodels
+tqdm

a3em_analysis-0.0.1/src/a3em/utils.py

@@ -0,0 +1,108 @@
+import librosa
+import numpy as np
+import os
+from pathlib import Path
+from scipy.signal import butter, sosfilt
+
+
+def preprocess(audio: np.ndarray, sample_rate: int, normalization: float = 0.7):
+    # parameter validation
+    if normalization < 0.0 or normalization > 1.0:
+        raise ValueError('the normalization factor must be between 0.0 and 1.0')
+
+    # high pass filter
+    high_pass = butter(2, 15, btype='highpass', fs=sample_rate, output='sos')
+    audio = sosfilt(high_pass, audio)
+
+    # low pass filter
+    low_pass = butter(2, 200, btype='lowpass', fs=sample_rate, output='sos')
+    audio = sosfilt(low_pass, audio)
+
+    # normalize 70%
+    audio = audio / np.max(np.abs(audio)) * normalization
+
+    return audio
+
+
+def extract_features(audio: np.ndarray, sample_rate: int) -> float:
+    # spectrograms
+    S = np.abs(librosa.stft(audio))**2
+    power_per_freq = S.mean(axis=1)
+    S_mel = librosa.feature.melspectrogram(y=audio, sr=sample_rate, n_mels=26)
+
+    #peak freq
+    freqs  = librosa.fft_frequencies(sr=sample_rate)
+    freq_mask = (freqs > 15) & (freqs < 60)
+    peak_freq = freqs[freq_mask][np.argmax(power_per_freq[freq_mask])]
+
+    #centroid
+    centroid = librosa.feature.spectral_centroid(y=audio, sr=sample_rate)
+    mean_centroid = centroid.mean()
+
+    # bandwidth
+    bandwidth = librosa.feature.spectral_bandwidth(y=audio, sr=sample_rate)
+    mean_bandwidth = bandwidth.mean()
+
+    # 5% and 95% freqs
+    cumulative = np.cumsum(power_per_freq)
+    cumulative = cumulative /cumulative[-1]
+    freq_5 = freqs[np.searchsorted(cumulative, 0.05)]
+    freq_95 = freqs[np.searchsorted(cumulative, 0.95)]
+
+    # MFCCS
+    mfccs = librosa.feature.mfcc(y=audio, sr=sample_rate, n_mfcc=13)
+    mfcc_dict = {f'mfcc_{i+1}': mfccs[i].mean() for i in range(13)}
+
+    mel_means = S_mel.mean(axis=1)
+    mel_dict = {f'mel_mean_{i+1}': v for i, v in enumerate(mel_means)}
+
+    # TODO - spectral flatness
+
+    # Harmonic-to-noise ratio (HPSS approximation)
+    harmonic, percussive = librosa.effects.hpss(audio)
+
+    harmonic_energy = np.sum(harmonic ** 2)
+    noise_energy = np.sum(percussive ** 2)
+
+    hnr = 10 * np.log10(
+        (harmonic_energy + 1e-10) /
+        (noise_energy + 1e-10)
+    )
+
+    # Low-frequency harmonic-to-noise ratio
+    low_pass = butter(2, 60, btype='lowpass', fs=sample_rate, output='sos')
+    low_audio = sosfilt(low_pass, audio)
+
+    harmonic, percussive = librosa.effects.hpss(low_audio)
+
+    harmonic_energy = np.sum(harmonic ** 2)
+    noise_energy = np.sum(percussive ** 2)
+
+    hnr_low = 10 * np.log10(
+        (harmonic_energy + 1e-10) /
+        (noise_energy + 1e-10)
+    )
+
+    return {
+        'peak_freq': peak_freq,
+        'centroid': mean_centroid,
+        'bandwidth': mean_bandwidth,
+        'freq_5': freq_5,
+        'freq_95': freq_95,
+        'hnr': hnr,
+        'hnr_low': hnr_low,
+        **mfcc_dict,
+        **mel_dict
+    }
+
+
+def clean_directory(path: Path, base_path: bool = True):
+    contents = list(path.iterdir())
+    while len(contents) != 0:
+        if contents[0].is_dir():
+            clean_directory(contents[0], base_path=False)
+        else:
+            contents[0].unlink()
+        contents.pop(0)
+    if not base_path:
+        path.rmdir()