mbari-pbp 0.3.0__tar.gz

mbari_pbp-0.3.0/PKG-INFO

@@ -0,0 +1,87 @@
+Metadata-Version: 2.1
+Name: mbari-pbp
+Version: 0.3.0
+Summary: PyPAM based Processing
+License: MIT
+Author: Carlos Rueda
+Author-email: carueda@mbari.org
+Requires-Python: >=3.9,<3.12.0
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Requires-Dist: Pyarrow (==15.0.0)
+Requires-Dist: boto3 (==1.34.47)
+Requires-Dist: dataclasses-json (==0.6.4)
+Requires-Dist: google-cloud-storage (==2.14.0)
+Requires-Dist: h5netcdf (==1.3.0)
+Requires-Dist: lifewatch-pypam (==0.3.0)
+Requires-Dist: marshmallow (==3.20.2)
+Requires-Dist: progressbar2 (==3.53.1)
+Requires-Dist: pvlib (==0.10.3)
+Requires-Dist: python-dateutil (==2.8.2)
+Requires-Dist: pyyaml (==6.0.1)
+Requires-Dist: soundfile (==0.12.1)
+Description-Content-Type: text/markdown
+
+# PyPAM based data processing
+
+This package uses [PyPAM](https://github.com/lifewatch/pypam/)
+to generate _hybrid millidecade band spectra_ for soundscape data.
+
+**Status**: Functional version, including support for S3-based cloud based processing.
+
+- [x] JSON generation of timekeeping
+- [x] Timekeeping based on given JSON indicating start and duration of every available (recognized) sound file
+- [x] Audio file processing
+    - [x] Frequency and psd array output
+    - [x] Concatenation of processed 1-minute segments for daily product
+    - [x] Calibration with given sensitivity file (NetCDF)
+    - [x] Calibration with given flat sensitivity value
+- [x] Data products
+    - [x] NetCDF with metadata
+    - [x] CSV (optional)
+    - [x] Summary plot (optional)
+- [x] Cloud processing (inputs downloaded from, and generated products uploaded to S3)
+
+TODO more details
+
+## Setup
+
+### Create and activate virtual environment
+
+    python3.9 -m venv virtenv
+    source virtenv/bin/activate
+
+### Install dependencies
+
+    pip3 install -r requirements.txt
+
+### Programs
+
+- `src/main.py` - Main CLI program, run `python src/main.py --help` for usage.
+
+- `src/main_cloud.py` - Main program for cloud based processing. 
+   All parameters passed via environment variables, see source file.
+
+- `src/plot.py` - Plotting program: `python src/plot.py --help`.
+ 
+- `src/main_json_generator.py` - Main CLI program to generate JSONS with audio metadata, run `python src/main_json_gen.py --help` for usage.
+
+## Refs
+
+- PyPAM - Python tool for Passive Acoustic Monitoring –
+  <https://doi.org/10.5281/zenodo.6044593>
+- Computation of single-sided mean-square sound pressure spectral density with 1 Hz resolution follows
+  ISO 18405 3.1.3.13 (International Standard ISO 18405:2017(E), Underwater Acoustics – Terminology. Geneva: ISO)
+  – https://www.iso.org/standard/62406.html
+- Hybrid millidecade spectra: A practical format for exchange of long-term ambient sound data –
+  <https://asa.scitation.org/doi/10.1121/10.0003324>
+- Erratum: Hybrid millidecade spectra –
+  <https://asa.scitation.org/doi/10.1121/10.0005818>
+
+## Development
+
+See [DEVEL.md](DEVEL.md) for details.
+

mbari_pbp-0.3.0/README.md

@@ -0,0 +1,59 @@
+# PyPAM based data processing
+
+This package uses [PyPAM](https://github.com/lifewatch/pypam/)
+to generate _hybrid millidecade band spectra_ for soundscape data.
+
+**Status**: Functional version, including support for S3-based cloud based processing.
+
+- [x] JSON generation of timekeeping
+- [x] Timekeeping based on given JSON indicating start and duration of every available (recognized) sound file
+- [x] Audio file processing
+    - [x] Frequency and psd array output
+    - [x] Concatenation of processed 1-minute segments for daily product
+    - [x] Calibration with given sensitivity file (NetCDF)
+    - [x] Calibration with given flat sensitivity value
+- [x] Data products
+    - [x] NetCDF with metadata
+    - [x] CSV (optional)
+    - [x] Summary plot (optional)
+- [x] Cloud processing (inputs downloaded from, and generated products uploaded to S3)
+
+TODO more details
+
+## Setup
+
+### Create and activate virtual environment
+
+    python3.9 -m venv virtenv
+    source virtenv/bin/activate
+
+### Install dependencies
+
+    pip3 install -r requirements.txt
+
+### Programs
+
+- `src/main.py` - Main CLI program, run `python src/main.py --help` for usage.
+
+- `src/main_cloud.py` - Main program for cloud based processing. 
+   All parameters passed via environment variables, see source file.
+
+- `src/plot.py` - Plotting program: `python src/plot.py --help`.
+ 
+- `src/main_json_generator.py` - Main CLI program to generate JSONS with audio metadata, run `python src/main_json_gen.py --help` for usage.
+
+## Refs
+
+- PyPAM - Python tool for Passive Acoustic Monitoring –
+  <https://doi.org/10.5281/zenodo.6044593>
+- Computation of single-sided mean-square sound pressure spectral density with 1 Hz resolution follows
+  ISO 18405 3.1.3.13 (International Standard ISO 18405:2017(E), Underwater Acoustics – Terminology. Geneva: ISO)
+  – https://www.iso.org/standard/62406.html
+- Hybrid millidecade spectra: A practical format for exchange of long-term ambient sound data –
+  <https://asa.scitation.org/doi/10.1121/10.0003324>
+- Erratum: Hybrid millidecade spectra –
+  <https://asa.scitation.org/doi/10.1121/10.0005818>
+
+## Development
+
+See [DEVEL.md](DEVEL.md) for details.

mbari_pbp-0.3.0/pbp/__init__.py

@@ -0,0 +1,49 @@
+import os
+
+import xarray as xr
+
+from pbp.logging_helper import PbpLogger
+
+
+def save_dataset_to_netcdf(logger: PbpLogger, ds: xr.Dataset, filename: str) -> bool:
+    logger.info(f"  - saving dataset to: {filename}")
+    try:
+        ds.to_netcdf(
+            filename,
+            engine="h5netcdf",
+            encoding={
+                "effort": {"_FillValue": None},
+                "frequency": {"_FillValue": None},
+                "sensitivity": {"_FillValue": None},
+            },
+        )
+        return True
+    except Exception as e:  # pylint: disable=broad-exception-caught
+        error = f"Unable to save {filename}: {e}"
+        logger.error(error)
+        print(error)
+        return False
+
+
+def save_dataset_to_csv(logger: PbpLogger, ds: xr.Dataset, filename: str):
+    logger.info(f"  - saving dataset to: {filename}")
+    try:
+        ds.to_pandas().to_csv(filename, float_format="%.1f")
+    except Exception as e:  # pylint: disable=broad-exception-caught
+        logger.error(f"Unable to save {filename}: {e}")
+
+
+def get_cpus_to_use(num_cpus: int) -> int:
+    cpu_count: int = os.cpu_count() or 1
+    if num_cpus <= 0 or num_cpus > cpu_count:
+        num_cpus = cpu_count
+    return num_cpus
+
+
+class PBPException(Exception):
+    """
+    Placeholder for a more specific exception.
+    """
+
+    def __init__(self, msg: str):
+        super().__init__(f"PBPException({msg})")