sonusai 0.15.9__py3-none-any.whl → 0.16.0__py3-none-any.whl

sonusai/utils/asr_manifest_functions/__init__.py

@@ -4,3 +4,4 @@ from .librispeech import collect_librispeech_transcripts
 from .librispeech import get_librispeech_manifest_entry
 from .vctk_noisy_speech import collect_vctk_noisy_speech_transcripts
 from .vctk_noisy_speech import get_vctk_noisy_speech_manifest_entry
+from .mcgill_speech import get_mcgill_speech_manifest_entry

sonusai/utils/asr_manifest_functions/mcgill_speech.py

@@ -0,0 +1,29 @@
+from sonusai.utils.asr_manifest_functions import PathInfo
+
+
+def get_mcgill_speech_manifest_entry(entry: PathInfo, transcript_data: list[str]) -> dict:
+    from os.path import splitext
+    from os.path import basename
+    from subprocess import check_output
+
+    from sonusai import SonusAIError
+
+    name = splitext(entry.abs_path)[0]
+    duration = float(check_output(f'soxi -D {entry.abs_path}', shell=True))
+    # i.e., from MA01_02.wav, get 01_02
+    promptname = basename(name)[2:]
+    # paragraph num
+    pnum = int(promptname[0:2])
+    snum = int(promptname[3:5])
+    idx = 11 * (pnum - 1) + (snum - 1)
+    try:
+        # remove prompt-id prefix and \n suffix
+        text = transcript_data[idx][6:-1]
+    except IndexError:
+        raise SonusAIError(f'Could not find {promptname}, idx {idx} in transcript data')
+
+    return {
+        'audio_filepath': entry.audio_filepath,
+        'text':           text,
+        'duration':       duration,
+    }

sonusai/utils/{trim_docstring.py → docstring.py}

@@ -28,3 +28,23 @@ def trim_docstring(docstring: str) -> str:
 
     # Return a single string
     return '\n'.join(trimmed)
+
+
+def add_commands_to_docstring(docstring: str, plugin_docstrings: list[str]) -> str:
+    """Add commands to docstring"""
+    import sonusai
+
+    lines = docstring.splitlines()
+
+    start = lines.index('The sonusai commands are:')
+    end = lines.index('', start)
+
+    commands = sonusai.commands_doc.splitlines()
+    for plugin_docstring in plugin_docstrings:
+        commands.extend(plugin_docstring.splitlines())
+    commands.sort()
+    commands = list(filter(None, commands))
+
+    lines = lines[:start + 1] + commands + lines[end:]
+
+    return '\n'.join(lines)

sonusai/utils/model_utils.py

@@ -0,0 +1,30 @@
+from typing import Any
+
+
+def import_module(name: str) -> Any:
+    """Import a Python module adding the module file's directory to the Python system path so that relative package
+    imports are found correctly.
+    """
+    import os
+    import sys
+    from importlib import import_module
+
+    from sonusai import SonusAIError
+
+    try:
+        path = os.path.dirname(name)
+        if len(path) < 1:
+            path = './'
+
+        # Add model file location to system path
+        sys.path.append(os.path.abspath(path))
+
+        try:
+            root = os.path.splitext(os.path.basename(name))[0]
+            model = import_module(root)
+        except Exception as e:
+            raise SonusAIError(f'Error: could not import model from {name}: {e}.')
+    except Exception as e:
+        raise SonusAIError(f'Error: could not find {name}: {e}.')
+
+    return model

sonusai/utils/onnx_utils.py

@@ -3,33 +3,15 @@ from dataclasses import dataclass
 from onnxruntime import InferenceSession
 
 
-def replace_stateful_grus(keras_model, onnx_model):
-    """Replace stateful GRUs with custom layers."""
-    import warnings
-
-    with warnings.catch_warnings():
-        warnings.simplefilter('ignore')
-        from keras.layers import GRU
-
-    stateful_gru_names = []
-    for i in range(len(keras_model.layers)):
-        layer = keras_model.layers[i]
-        if isinstance(layer, GRU):
-            if layer.stateful:
-                stateful_gru_names.append(layer.name)
-
-    for node_index in range(len(onnx_model.graph.node)):
-        node = onnx_model.graph.node[node_index]
-        replace = False
-        if node.op_type == 'GRU':
-            for i in node.input:
-                for n in stateful_gru_names:
-                    if n in i:
-                        replace = True
-        if node.name in stateful_gru_names or replace:
-            node.op_type = 'SGRU'
-
-    return onnx_model
+@dataclass(frozen=True)
+class SonusAIMetaData:
+    input_shape: list[int]
+    output_shape: list[int]
+    flattened: bool
+    timestep: bool
+    channel: bool
+    mutex: bool
+    feature: str
 
 
 def add_sonusai_metadata(model,
@@ -38,13 +20,14 @@ def add_sonusai_metadata(model,
                          has_channel: bool = False,
                          is_mutex: bool = True,
                          feature: str = ''):
-    """Add SonusAI metadata to ONNX model.
-      model           keras model
-      is_flattened    model feature data is flattened
-      has_timestep    model has timestep dimension
-      has_channel     model has channel dimension
-      is_mutex        model label output is mutually exclusive
-      feature         model feature type
+    """Add SonusAI metadata to an ONNX model.
+
+    :param model: ONNX model
+    :param is_flattened: Model feature data is flattened
+    :param has_timestep: Model has timestep dimension
+    :param has_channel: Model has channel dimension
+    :param is_mutex: Model label output is mutually exclusive
+    :param feature: Model feature type
     """
     is_flattened_flag = model.metadata_props.add()
     is_flattened_flag.key = 'is_flattened'
@@ -69,18 +52,9 @@ def add_sonusai_metadata(model,
     return model
 
 
-@dataclass(frozen=True)
-class SonusAIMetaData:
-    input_shape: list[int]
-    output_shape: list[int]
-    flattened: bool
-    timestep: bool
-    channel: bool
-    mutex: bool
-    feature: str
-
-
 def get_sonusai_metadata(model: InferenceSession) -> SonusAIMetaData:
+    """Get SonusAI metadata from an ONNX model.
+    """
     m = model.get_modelmeta().custom_metadata_map
     return SonusAIMetaData(input_shape=model.get_inputs()[0].shape,
                            output_shape=model.get_outputs()[0].shape,

{sonusai-0.15.9.dist-info → sonusai-0.16.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sonusai
-Version: 0.15.9
+Version: 0.16.0
 Summary: Framework for building deep neural network models for sound, speech, and voice AI
 Home-page: https://aaware.com
 License: GPL-3.0-only
@@ -15,57 +15,39 @@ Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Requires-Dist: PyYAML (>=6.0.1,<7.0.0)
-Requires-Dist: aixplain (>=0.2.6,<0.3.0)
-Requires-Dist: bitarray (>=2.9.2,<3.0.0)
-Requires-Dist: ctranslate2 (==4.1.0)
 Requires-Dist: dataclasses-json (>=0.6.1,<0.7.0)
-Requires-Dist: deepgram-sdk (>=3.0.0,<4.0.0)
 Requires-Dist: docopt (>=0.6.2,<0.7.0)
-Requires-Dist: einops (>=0.7.0,<0.8.0)
-Requires-Dist: faster-whisper (>=1.0.1,<2.0.0)
-Requires-Dist: geomloss (>=0.2.6,<0.3.0)
 Requires-Dist: h5py (>=3.11.0,<4.0.0)
-Requires-Dist: hydra-core (>=1.3.2,<2.0.0)
 Requires-Dist: jiwer (>=3.0.3,<4.0.0)
-Requires-Dist: keras (>=3.1.1,<4.0.0)
-Requires-Dist: keras-tuner (>=1.4.7,<2.0.0)
 Requires-Dist: librosa (>=0.10.1,<0.11.0)
-Requires-Dist: lightning (>=2.2,<2.3)
 Requires-Dist: matplotlib (>=3.8.0,<4.0.0)
-Requires-Dist: omegaconf (>=2.3.0,<3.0.0)
 Requires-Dist: onnx (>=1.14.1,<2.0.0)
 Requires-Dist: onnxruntime (>=1.16.1,<2.0.0)
 Requires-Dist: paho-mqtt (>=2.0.0,<3.0.0)
 Requires-Dist: pandas (>=2.1.1,<3.0.0)
 Requires-Dist: pesq (>=0.0.4,<0.0.5)
-Requires-Dist: pyaaware (>=1.5.3,<2.0.0)
+Requires-Dist: pyaaware (>=1.5.7,<2.0.0)
 Requires-Dist: pyaudio (>=0.2.14,<0.3.0)
 Requires-Dist: pydub (>=0.25.1,<0.26.0)
 Requires-Dist: pystoi (>=0.4.0,<0.5.0)
-Requires-Dist: python-magic (>=0.4.27,<0.5.0)
 Requires-Dist: requests (>=2.31.0,<3.0.0)
-Requires-Dist: sacrebleu (>=2.4.2,<3.0.0)
 Requires-Dist: samplerate (>=0.2.1,<0.3.0)
 Requires-Dist: soundfile (>=0.12.1,<0.13.0)
 Requires-Dist: sox (>=1.4.1,<2.0.0)
-Requires-Dist: speechrecognition (>=3.10.1,<4.0.0)
-Requires-Dist: tensorflow (>=2.15.0,<3.0.0)
-Requires-Dist: tf2onnx (>=1.15.1,<2.0.0)
 Requires-Dist: torch (>=2.2,<2.3)
 Requires-Dist: torchaudio (>=2.2,<2.3)
-Requires-Dist: torchinfo (>=1.8.0,<2.0.0)
 Requires-Dist: tqdm (>=4.66.1,<5.0.0)
 Description-Content-Type: text/x-rst
 
-Sonus AI: Framework for simplified creation of deep NN models for sound, speech, and voice AI
+SonusAI: Framework for simplified creation of deep NN models for sound, speech, and voice AI
 
-Sonus AI includes functions for pre-processing training and validation data and
+SonusAI includes functions for pre-processing training and validation data and
 creating performance metrics reports for key types of Keras models:
 - recurrent, convolutional, or a combination (i.e. RCNNs)
 - binary, multiclass single-label, multiclass multi-label, and regression
 - training with data augmentations:  noise mixing, pitch and time stretch, etc.
 
-Sonus AI python functions are used by:
- - Aaware Inc. sonusai executable:  Easily create train/validation data, run prediction, evaluate model performance
- - Keras model scripts:             User python scripts for keras model creation, training, and prediction. These can use sonusai-specific data but also some general useful utilities for trainining rnn-based models like CRNN's, DSCRNN's, etc. in Keras
+SonusAI python functions are used by:
+ - Aaware Inc. sonusai framework:   Easily create train/validation data, run prediction, evaluate model performance
+ - Keras model scripts:             User python scripts for Keras model creation, training, and prediction. These can use sonusai-specific data but also some general useful utilities for training rnn-based models like CRNN's, DSCRNN's, etc. in Keras.
 

{sonusai-0.15.9.dist-info → sonusai-0.16.0.dist-info}/RECORD

@@ -1,27 +1,20 @@
-sonusai/__init__.py,sha256=KmIJ9wni9d9v5pyu0pUxbacZIHGkAywB9CJwl7JME28,1526
+sonusai/__init__.py,sha256=ziSNMMCNS-tIp2ON1e7ST_xNLT-P8BvJfV2IxUvSJZs,2896
 sonusai/aawscd_probwrite.py,sha256=GukR5owp_0A3DrqSl9fHWULYgclNft4D5OkHIwfxxkc,3698
-sonusai/audiofe.py,sha256=XE_cgOhhTryjPUePxW_8NY1TwrnRZ6BHCsH-gp8PmYw,11471
-sonusai/calc_metric_spenh.py,sha256=D8iQVSIhFhrsUwKuIP-S38NBnyfAOZlsOIIgOZwGOOI,60852
+sonusai/audiofe.py,sha256=v-GWrma5NNSukkPH427_6ilvFghUlDy4N9sfp_gvwCY,9303
+sonusai/calc_metric_spenh.py,sha256=zJD9cuO4iXvOTuqDjpv-tJQg9bC7r5L7K74Av16lOk4,61762
 sonusai/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sonusai/data/genmixdb.yml,sha256=-XSs_hUR6wHJVoTPmSewzXL7u61X-xmHY46lNPatxSE,1025
 sonusai/data/speech_ma01_01.wav,sha256=PK0vMKg-NR6rPE3KouxHGF6PKXnJCr7AwjMqfu98LUA,76644
 sonusai/data/whitenoise.wav,sha256=I2umov0m34y56F9IsIBi1XtE76ZeZaSKDf70cJRe3pI,1920044
-sonusai/data_generator/__init__.py,sha256=ouCpY5EDV35fKFeKGQfIcU8uE-c3QcuNerTxUA1X5L8,232
-sonusai/data_generator/dataset_from_mixdb.py,sha256=D14L8BL7a0WgkF8a8eogQ9Hk9ow4_RK3QBGsZ-HDAog,5493
-sonusai/data_generator/keras_from_mixdb.py,sha256=14r89aX6Dr9ZKsmMRC7HDXbJrPrCZC1liwwLmZUKj0w,6182
-sonusai/data_generator/torch_from_mixdb.py,sha256=lvEe9DDu_rIaoyhv9PW4UAnAWp5N74L8kRfxUhsh7oo,4279
 sonusai/doc/__init__.py,sha256=rP5Hgn0Iys_xkuv4caxngdqehuU4zLZsiKuv8Nde67M,19
 sonusai/doc/doc.py,sha256=3z210v6ZckuOlsGZ3ySQBdlCNmBp2M1ahqhqG_eUN58,22664
 sonusai/doc.py,sha256=l8CaFgLI8mqx4tn0aXfxKqa2dy9GgC0zjYxZAkpmi1E,878
 sonusai/genft.py,sha256=6hOds8d-pYRawesLYh7XLrQh4PweWUj8U5Cbzx45bNQ,5572
 sonusai/genmix.py,sha256=0AiUfF7n0CGOp5v-woNfeP3-QaVQUb0CJZc0oXkvqpk,7016
-sonusai/genmixdb.py,sha256=rAxCKNPkOXaAugEfp9pTcHCQONapdTnxMlBsIPIoizE,19639
+sonusai/genmixdb.py,sha256=b-c22X7dd1ePLGrVuhVcn-AXHnxSrC5lTaR6KlrD1-4,19610
 sonusai/gentcst.py,sha256=8jeXirgJe0OGgknC8A-rIudjHeH8UTYPpuh71Ha-I3w,20165
-sonusai/keras_onnx.py,sha256=WHcPIcff4VPdiXqGX-TU-_x-UuKUD3nNpQtCX-2NEVQ,2658
-sonusai/keras_predict.py,sha256=_83EtPtnfrqwUzC2H2tk4LI90RiQdyEEBxFGTgFPl3M,9090
-sonusai/keras_train.py,sha256=8_M5vY-CkonPzbOtOF3Vk-wox-42o8fkaOKLjk7Oc2k,13226
 sonusai/lsdb.py,sha256=TTMQ-0H8fFzUSczt6yjy-9xUjZSdIGQzTVH5Xr6XPSA,5941
-sonusai/main.py,sha256=KjN0dCI6rWare4wo_ACzTlURW7pvTw03n51pH7EyLAU,3108
+sonusai/main.py,sha256=GC-pQrSqx9tWwIcmEo6V9SraEv5KskBLS_W_wz-f2ZM,2509
 sonusai/metrics/__init__.py,sha256=56itZW3S1I7ZYvbxPmFIVPAh1AIJZdljByz1uCrHqFE,635
 sonusai/metrics/calc_class_weights.py,sha256=dyY7daEIf5Ms5tfTf6wF0fkx_GnMADHOZR_rtsfGoVM,3933
 sonusai/metrics/calc_optimal_thresholds.py,sha256=9fRfwl-aKAbzHJyqGHv4o8BpZXG9HHB7zUJObHXfYM4,3522
@@ -35,11 +28,11 @@ sonusai/metrics/class_summary.py,sha256=4Mb25nuk6eqotnQSFMuOQL3zofGcpNXDfDlPa513
 sonusai/metrics/confusion_matrix_summary.py,sha256=3qg6TMKjJeHtNjj2YnNjPFSlMrQXt0Zcu1dLkGB_aPU,4001
 sonusai/metrics/one_hot.py,sha256=QSeH_GdqBpOAKLrNnQ8gjcPC-vSdUqC0yPEQueTA6VI,13548
 sonusai/metrics/snr_summary.py,sha256=P4U5_Xr7v9F8kF-rZBnpsVNt3p42rIVS6zmch8yfVfg,5575
-sonusai/mixture/__init__.py,sha256=fCVSlizYxUUQQD9nSZ8bEbfc_TB2yiOC14HPOB4KFz4,5287
+sonusai/mixture/__init__.py,sha256=BfSJL91URq8-JDlbtpc5SQoQRWEUXvxKozbuquX4Mok,5326
 sonusai/mixture/audio.py,sha256=S-ZROf5rVvwv1TCEuwJHz1FfX4oVubb4QhbybUMMqtM,2150
 sonusai/mixture/augmentation.py,sha256=Blb90tdTwBOj5w9tRcYyS5H67YJuFiXsGqwZWd7ON4g,10468
 sonusai/mixture/class_count.py,sha256=_wFnVl2yEOnbor7pLg7cYOUeX6nioov-03Cv3SEbh2k,996
-sonusai/mixture/config.py,sha256=CXIkVRJmaW2QW_sGl0aIqPf7I_TesyGhUYzxouw5UX4,22266
+sonusai/mixture/config.py,sha256=d2IzZ1samHWGMpoKzSmUwMyAWWhgmyNoxyO8oiUwbsg,22193
 sonusai/mixture/constants.py,sha256=xjCskcQi6khqYZDf7j6z1OkeN1C6wE06kBBapcJiNI4,1428
 sonusai/mixture/datatypes.py,sha256=zaxfOHw8ddt-i8JPYOPnlqWz_EHBEDoO4q2VAqJViHM,8173
 sonusai/mixture/eq_rule_is_valid.py,sha256=MpQwRA5M76wSiQWEI1lW2cLFdPaMttBLcQp3tWD8efM,1243
@@ -67,31 +60,23 @@ sonusai/mixture/truth_functions/file.py,sha256=jOJuC_3y9BH6GGOp9eKcbVrHLVRzUA80B
 sonusai/mixture/truth_functions/phoneme.py,sha256=stYdlPuNytQK_LLT61OJLfYSqKd-sDjQZdtJKGzt5wA,479
 sonusai/mixture/truth_functions/sed.py,sha256=8cHjEFjZaH_0hIOHhPmj4AJz2GpEADM6Ys2x4NoiWSY,2469
 sonusai/mixture/truth_functions/target.py,sha256=KAsjugDRooOA5BRcHVAbZRgV7l8S5CFg7CZ0XtKZaQ0,5764
-sonusai/mkmanifest.py,sha256=dIPVFKKhnhHdq63OGr6p__pK7fyx3OdKVtbmGUJxsR8,7078
+sonusai/mkmanifest.py,sha256=r6bQ6uoBVJIAviqNtJo7AZjXS8EhxdSAW-NHlhPzDCU,8646
 sonusai/mkwav.py,sha256=LZNyhq4gJEs_NtGvRsYHA2qfgkkODpt6HoH1b-Tjjuw,5266
 sonusai/onnx_predict.py,sha256=Bz_pR28oAZBarNajlKwyzBxmW7ktum77SmxDN2onKPM,9060
 sonusai/plot.py,sha256=u-PvF8guNcm0b-GN99xfEkrcAAtidAEY3RLDzNvcyYk,17014
 sonusai/post_spenh_targetf.py,sha256=NIMhDXeDuUqeWukNaAUMvDw9JpEVCauwjrL2F4M9nrI,4927
 sonusai/queries/__init__.py,sha256=oKY5JeqZ4Cz7DwCwPc1_ydB8bUs6KaMcWFp_w02TjOs,255
 sonusai/queries/queries.py,sha256=FNMUKnoY_Ya9S5sNhsB8ppwy0B7V55ilbbjhQRv_UN8,7552
-sonusai/torchl_onnx.py,sha256=5JYow3XpBaUdtuyAW0mOZyCKL_4FrHvEekYBRdDT6KA,8967
-sonusai/torchl_predict.py,sha256=P1ySDH_ITOPefZ2xZqyxyIrsNDqblKTBLZqFApgo5EU,26238
-sonusai/torchl_train.py,sha256=NPCRB0gwTvabivmOz78gjUreDeO1z16PYuw7L1-pIRQ,9680
 sonusai/tplot.py,sha256=yFyyyg9ymp2Eh-64Muu0EFFEY61MoJSV0a_fy9OWaCk,14485
-sonusai/utils/__init__.py,sha256=tVSmxinSo0Enexpol6wCzz6tU7WrueC-YslFgQr-o7M,2382
-sonusai/utils/asl_p56.py,sha256=GCKlz-NLInQ0z41XBi0mOvGdSfRZf3WI53necVNDo80,3837
-sonusai/utils/asr.py,sha256=QN1wdO9-EqD72-ixr4lnzsPfT8i0syhTGj1evKNJWe4,2021
-sonusai/utils/asr_functions/__init__.py,sha256=4boXXOXlQHTt8K2DWOwFXSlc8D2NLFd8QTc68yL2ejU,214
-sonusai/utils/asr_functions/aaware_whisper.py,sha256=6JnF8-a-39dYk4gVILWRUD5Ou98T6wPe9g4m8c6MF1A,1994
-sonusai/utils/asr_functions/aixplain_whisper.py,sha256=Fdg3z4HSw1zBu2CQGVU-e8xM3PLTh1Vn4c6fawBOWwQ,1694
-sonusai/utils/asr_functions/data.py,sha256=FQn-y63aOUooOjgKtkzkpFz5Gm9dwn3JPwcR0gC7fYA,418
-sonusai/utils/asr_functions/deepgram.py,sha256=sxFRDJn5EjSLv01G2lMoxXFHzXqFnA9ln4VcQWqGP80,5237
-sonusai/utils/asr_functions/fastwhisper.py,sha256=cyjC3U4UmBZstIvmpuUbNfrcgFFQd1-FLipDaK3_IpU,2856
-sonusai/utils/asr_functions/google.py,sha256=swzEKdod6EwGUKX8LJYcZPf6hJpuvsp9N6QJi7ip0yQ,3320
-sonusai/utils/asr_functions/whisper.py,sha256=IuOodbJY1RsiS3o18cVuBcIXn28TCGQz2BCFR1Up08s,1033
-sonusai/utils/asr_manifest_functions/__init__.py,sha256=Lz12aCGvfngZkLoUxHSqFjHc4wig4ZjQyFbWPWysgrI,310
+sonusai/utils/__init__.py,sha256=TCXlcW8W0Up2f5ciSgz3DabvH1MxrrWD0LK6pQTJkeA,2215
+sonusai/utils/asl_p56.py,sha256=-bvQpd-jRQVURbkZJpRoyEAq6gTv9Rc3oFDbh5_lcjY,3861
+sonusai/utils/asr.py,sha256=6y6VYJizHpuQ3MgKbEQ4t2gofO-MW6Ez23oAd6d23IE,2920
+sonusai/utils/asr_functions/__init__.py,sha256=JyHK67s97bw7QzrlkboWhws4yNytdPatqzLJxfwx-yw,43
+sonusai/utils/asr_functions/aaware_whisper.py,sha256=LzO9CZV0wBWkjmCR2nSWN_AW9UJwriAsC1OYSlfVeT8,1981
+sonusai/utils/asr_manifest_functions/__init__.py,sha256=V-w4R7SHUyoeDuMR3tS12j6DGhmfTlUibMKgq1c6ga0,370
 sonusai/utils/asr_manifest_functions/data.py,sha256=mJsaHccBReguOJu9qsshRhL-3GbeyqM0-PXMseFnZbE,151
 sonusai/utils/asr_manifest_functions/librispeech.py,sha256=HIaytcYmjRUkuR6fCQlv3Jh3IDWSox_A6WFcFFAHN9M,1635
+sonusai/utils/asr_manifest_functions/mcgill_speech.py,sha256=2uF9qgBwcue9W9dlRo16Kvr3jX5g53AGjW2rwob-Cqk,957
 sonusai/utils/asr_manifest_functions/vctk_noisy_speech.py,sha256=-69lM0dz18KbU5_-dmSeqDoNNwgJj4UlxgGkNBEi3wM,2169
 sonusai/utils/audio_devices.py,sha256=LgaXTln1oRArBzaet3rZiIO2plgtaThuGBc3sJ_sLlo,1414
 sonusai/utils/braced_glob.py,sha256=h4hab7YDbM4CjLg9iSzyHZrkd22IPUOY5zZqHdifkh8,1510
@@ -101,16 +86,17 @@ sonusai/utils/create_timestamp.py,sha256=TxoQXWZ3SFdBEHLOv-ujeIsTEJuiFnKOGRy-FQq
 sonusai/utils/create_ts_name.py,sha256=8RLKmgXwuGcbDMGgtTuc0MvGFfA7IOVqfjkE2T18GOo,405
 sonusai/utils/dataclass_from_dict.py,sha256=vAGnuMjhy0W9bxZ5usrH7mbQsFog3n0__IC4xyJyVUc,390
 sonusai/utils/db.py,sha256=lI77MJJLs4CTYxhjFUvBom2Kk2imAP34okOeO4irbDc,371
+sonusai/utils/docstring.py,sha256=JBecAq_a7KSzZ04tan0BlFA9SzhOQqY9A7Ue85kFQdU,1446
 sonusai/utils/energy_f.py,sha256=AsoGo3TCMnj9Kzqb9HUAYOL6vPGUseu1p-8z4LG-QYo,1462
 sonusai/utils/engineering_number.py,sha256=6lWRg-XW3FEllh-Zui1WHYcDnsBEY9cqcwcXPIs2-6o,5515
 sonusai/utils/get_frames_per_batch.py,sha256=xnq4tV7MT74N0H6b5ZsiAezqdXucboCLQw1Np9XpZbs,134
 sonusai/utils/get_label_names.py,sha256=bfFV_iFbXdtDp70zAz_CQQN9-QxBw_kt8vyAcr3TITM,828
 sonusai/utils/grouper.py,sha256=qyZ0nj84yOrC-RZsXHC-KJvcUliGktnV8S6-P3PD6_w,203
 sonusai/utils/human_readable_size.py,sha256=SjYT0fUlpbfCzCXHo6csir-VMwqfs5ogr-fgLCEqFk4,279
-sonusai/utils/keras_utils.py,sha256=1FJaaQE2yVYQqB6xTdgzjYKV_yvIA_hKob_uLOLf-0Y,8621
 sonusai/utils/max_text_width.py,sha256=pxiJMwb_zlkNntexgo7S6lAuF7NLLZvFdOCkxdsQJVY,315
+sonusai/utils/model_utils.py,sha256=lt2KOGJqsinG71W0i3U29UXFO-47GMAlEabsf2um7bA,862
 sonusai/utils/numeric_conversion.py,sha256=GRO_2Fba8CcxcFY7bEXKOEUEUX6neA-VN__Bxi1ULsE,340
-sonusai/utils/onnx_utils.py,sha256=A3JyMvjV_0c3YZFEUD8tkT4Er_R4p1P0jjgrfcFXnRQ,3037
+sonusai/utils/onnx_utils.py,sha256=BRsHGlcu5L0v_1z83MNy8TAcBeb7tJd_4OBJgOMLen8,2200
 sonusai/utils/parallel.py,sha256=bxedjCzBv9oxzU7NajRr6mOKmkCWr2P7FWAI0p2p9N8,1981
 sonusai/utils/print_mixture_details.py,sha256=BzYM4-wHHNa6zxPzBMUJxwKt0gKHmvbwdd7Yp0w15Yk,3017
 sonusai/utils/ranges.py,sha256=NPBZOVzMb95GTOIxltVO-wSzgcXqZ14wbdV46JDLKrw,1222
@@ -120,11 +106,10 @@ sonusai/utils/reshape.py,sha256=E8Eu6grynaeWwVO6peIR0BF22SrVaJSa1Rkl109lq6Y,5997
 sonusai/utils/seconds_to_hms.py,sha256=oxLuZhTJJr9swj-fOSOrZJ5vBNM7_BrOMQhX1pYpiv0,260
 sonusai/utils/stacked_complex.py,sha256=feLhz3GC1ILxBGMHOj3sJK--sidsXKbfwkalwAVwizc,2950
 sonusai/utils/stratified_shuffle_split.py,sha256=rJNXvBp-GxoKzH3OpL7k0ANSu5xMP2zJ7K1fm_33UzE,7022
-sonusai/utils/trim_docstring.py,sha256=dSrtiRsEN4wkkvKBp6WDr13RUypfqZzgH_jOBLs1ouY,881
 sonusai/utils/wave.py,sha256=O4ZXkZ6wjrKGa99wBCdFd8G6bp91MXXDnmGihpaEMh0,856
 sonusai/utils/yes_or_no.py,sha256=eMLXBVH0cEahiXY4W2KNORmwNQ-ba10eRtldh0y4NYg,263
 sonusai/vars.py,sha256=m2AefF0m5bXWGXpJj8Pi42zWL2ydeEj7bkak3GrtMyM,940
-sonusai-0.15.9.dist-info/METADATA,sha256=DudNQlTEQpWpzqyzyowz_V-J9epd7mrKgAYM6rFxaPo,3209
-sonusai-0.15.9.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-sonusai-0.15.9.dist-info/entry_points.txt,sha256=zMNjEphEPO6B3cD1GNpit7z-yA9tUU5-j3W2v-UWstU,92
-sonusai-0.15.9.dist-info/RECORD,,
+sonusai-0.16.0.dist-info/METADATA,sha256=MWrFI-GR33t2hVwlEwtLzzfEZiqO3mekRf7P7L8TaSk,2443
+sonusai-0.16.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+sonusai-0.16.0.dist-info/entry_points.txt,sha256=zMNjEphEPO6B3cD1GNpit7z-yA9tUU5-j3W2v-UWstU,92
+sonusai-0.16.0.dist-info/RECORD,,

sonusai/data_generator/__init__.py

@@ -1,5 +0,0 @@
-# SonusAI data generator classes
-from .keras_from_mixdb import KerasFromH5
-from .keras_from_mixdb import KerasFromMixtureDatabase
-from .torch_from_mixdb import AawareDataLoader
-from .torch_from_mixdb import TorchFromMixtureDatabase

sonusai/data_generator/dataset_from_mixdb.py

@@ -1,143 +0,0 @@
-import math
-import warnings
-from dataclasses import dataclass
-from typing import Optional
-
-import numpy as np
-
-from sonusai.mixture import GeneralizedIDs
-from sonusai.mixture import MixtureDatabase
-
-with warnings.catch_warnings():
-    warnings.simplefilter('ignore')
-    from keras.utils import Sequence
-
-
-class DatasetFromMixtureDatabase(Sequence):
-    """Generates data for Keras from a SonusAI mixture database
-    """
-
-    @dataclass(frozen=True)
-    class BatchParams:
-        mixids: list[int]
-        offset: int
-        extra: int
-        padding: int
-
-    def __init__(self,
-                 mixdb: MixtureDatabase,
-                 mixids: GeneralizedIDs,
-                 batch_size: int,
-                 timesteps: int,
-                 flatten: bool,
-                 add1ch: bool,
-                 shuffle: bool = False):
-        """Initialization
-        """
-        self.mixdb = mixdb
-        self.mixids = self.mixdb.mixids_to_list(mixids)
-        self.batch_size = batch_size
-        self.timesteps = timesteps
-        self.flatten = flatten
-        self.add1ch = add1ch
-        self.shuffle = shuffle
-        self.stride = self.mixdb.fg_stride
-        self.feature_parameters = self.mixdb.feature_parameters
-        self.num_classes = self.mixdb.num_classes
-        self.mixture_frame_segments = None
-        self.batch_frame_segments = None
-        self.total_batches: Optional[int] = None
-
-        self._initialize_mixtures()
-
-    def __len__(self) -> int:
-        """Denotes the number of batches per epoch
-        """
-        return self.total_batches
-
-    def __getitem__(self, batch_index: int) -> tuple[np.ndarray, np.ndarray]:
-        """Get one batch of data
-        """
-        from sonusai.utils import reshape_inputs
-
-        batch_params = self.batch_params[batch_index]
-
-        result = [self.mixdb.mixture_ft(mixid) for mixid in batch_params.mixids]
-        feature = np.vstack([result[i][0] for i in range(len(result))])
-        truth = np.vstack([result[i][1] for i in range(len(result))])
-
-        pad_shape = list(feature.shape)
-        pad_shape[0] = batch_params.padding
-        feature = np.vstack([feature, np.zeros(pad_shape)])
-
-        pad_shape = list(truth.shape)
-        pad_shape[0] = batch_params.padding
-        truth = np.vstack([truth, np.zeros(pad_shape)])
-
-        if batch_params.extra > 0:
-            feature = feature[batch_params.offset:-batch_params.extra]
-            truth = truth[batch_params.offset:-batch_params.extra]
-        else:
-            feature = feature[batch_params.offset:]
-            truth = truth[batch_params.offset:]
-
-        feature, truth = reshape_inputs(feature=feature,
-                                        truth=truth,
-                                        batch_size=self.batch_size,
-                                        timesteps=self.timesteps,
-                                        flatten=self.flatten,
-                                        add1ch=self.add1ch)
-
-        return feature, truth
-
-    def on_epoch_end(self) -> None:
-        """Modification of dataset between epochs
-        """
-        import random
-
-        if self.shuffle:
-            random.shuffle(self.mixids)
-            self._initialize_mixtures()
-
-    def _initialize_mixtures(self) -> None:
-        from sonusai.utils import get_frames_per_batch
-
-        frames_per_batch = get_frames_per_batch(self.batch_size, self.timesteps)
-        # Always extend the number of batches to use all available data
-        # The last batch may need padding
-        self.total_batches = math.ceil(self.mixdb.total_feature_frames(self.mixids) / frames_per_batch)
-
-        # Compute mixid, offset, and extra for dataset
-        # offsets and extras are needed because mixtures are not guaranteed to fall on batch boundaries.
-        # When fetching a new index that starts in the middle of a sequence of mixtures, the
-        # previous feature frame offset must be maintained in order to preserve the correct
-        # data sequence. And the extra must be maintained in order to preserve the correct data length.
-        cumulative_frames = 0
-        start_mixture_index = 0
-        offset = 0
-        self.batch_params = []
-        self.file_indices = []
-        total_frames = 0
-        for idx, mixid in enumerate(self.mixids):
-            current_frames = self.mixdb.mixture(mixid).samples // self.mixdb.feature_step_samples
-            self.file_indices.append(slice(total_frames, total_frames + current_frames))
-            total_frames += current_frames
-            cumulative_frames += current_frames
-            while cumulative_frames >= frames_per_batch:
-                extra = cumulative_frames - frames_per_batch
-                mixids = self.mixids[start_mixture_index:idx + 1]
-                self.batch_params.append(self.BatchParams(mixids=mixids, offset=offset, extra=extra, padding=0))
-                if extra == 0:
-                    start_mixture_index = idx + 1
-                    offset = 0
-                else:
-                    start_mixture_index = idx
-                    offset = current_frames - extra
-                cumulative_frames = extra
-
-        # If needed, add final batch with padding
-        needed_frames = self.total_batches * frames_per_batch
-        padding = needed_frames - total_frames
-        if padding != 0:
-            mixids = self.mixids[start_mixture_index:]
-            self.batch_params.append(self.BatchParams(mixids=mixids, offset=offset, extra=0, padding=padding))