sonusai 0.15.8__py3-none-any.whl → 0.16.0__py3-none-any.whl

sonusai/evaluate.py

@@ -1,245 +0,0 @@
-"""sonusai evaluate
-
-usage: evaluate [-hv] [-i MIXID] (-f FEATURE) (-p PREDICT) [-t PTHR] LOC
-
-options:
-    -h, --help
-    -v, --verbose                   Be verbose.
-    -i MIXID, --mixid MIXID         Mixture ID(s) to generate. [default: *].
-    -p PREDICT, --predict PREDICT   A directory containing prediction data.
-    -t PTHR, --thr PTHR             Optional prediction decision threshold(s). [default: 0].
-
-Evaluate calculates performance metrics of neural-network models from model prediction data and genft data.
-
-Inputs:
-    LOC         A SonusAI mixture database directory.
-    MIXID       A glob of mixture ID(s) to generate.
-    PREDICT     A directory containing SonusAI predict HDF5 files. Contains:
-                    dataset:    predict (either [frames, num_classes] or [frames, timesteps, num_classes])
-    PTHR        Scalar or array of thresholds. Default 0 will select values:
-                    argmax()    if mixdb indicates single-label mode (truth_mutex = true)
-                    0.5         if mixdb indicates multi-label mode  (truth_mutex = false)
-                If PTHR = -1, optimal thresholds are calculated using precision_recall_curve() which
-                optimizes F1 score.
-"""
-import numpy as np
-
-from sonusai import logger
-from sonusai.mixture import Feature
-from sonusai.mixture import MixtureDatabase
-from sonusai.mixture import Predict
-from sonusai.mixture import Segsnr
-from sonusai.mixture import Truth
-
-
-def evaluate(mixdb: MixtureDatabase,
-             truth: Truth,
-             predict: Predict = None,
-             segsnr: Segsnr = None,
-             output_dir: str = None,
-             predict_thr: float | np.ndarray = 0,
-             feature: Feature = None,
-             verbose: bool = False) -> None:
-    from os.path import join
-
-    from sonusai import initial_log_messages
-    from sonusai import update_console_handler
-    from sonusai.metrics import calc_optimal_thresholds
-    from sonusai.metrics import class_summary
-    from sonusai.metrics import snr_summary
-    from sonusai.mixture import SAMPLE_RATE
-    from sonusai.queries import get_mixids_from_snr
-    from sonusai.utils import get_num_classes_from_predict
-    from sonusai.utils import human_readable_size
-    from sonusai.utils import reshape_outputs
-    from sonusai.utils import seconds_to_hms
-
-    update_console_handler(verbose)
-    initial_log_messages('evaluate')
-
-    if truth.shape[-1] != predict.shape[-1]:
-        logger.exception(f'Number of classes in truth and predict are not equal. Exiting.')
-        raise SystemExit(1)
-
-    # truth, predict can be either [frames, num_classes] or [frames, timesteps, num_classes]
-    # in binary case dim may not exist, detect this and set num_classes == 1
-    timesteps = -1
-    predict, truth = reshape_outputs(predict=predict, truth=truth, timesteps=timesteps)
-    num_classes = get_num_classes_from_predict(predict=predict, timesteps=timesteps)
-
-    fdiff = truth.shape[0] - predict.shape[0]
-    if fdiff > 0:
-        # truth = truth[0:-fdiff,:]
-        predict = np.concatenate((predict, np.zeros((fdiff, num_classes), dtype=np.float32)))
-        logger.info(f'Truth has more feature-frames than predict, padding predict with zeros to match.')
-
-    if fdiff < 0:
-        predict = predict[0:fdiff, :]
-        logger.info(f'Predict has more feature-frames than truth, trimming predict to match.')
-
-    # Check segsnr, input is always in transform frames
-    compute_segsnr = False
-    if len(segsnr) > 0:
-        segsnr_feature_frames = segsnr.shape[0] / (mixdb.feature_step_samples / mixdb.ft_config.R)
-        if segsnr_feature_frames == truth.shape[0]:
-            compute_segsnr = True
-        else:
-            logger.warning('segsnr length does not match truth, ignoring.')
-
-    # Check predict_thr array or scalar and return final scalar predict_thr value
-    if not mixdb.truth_mutex:
-        if num_classes > 1:
-            if not isinstance(predict_thr, np.ndarray):
-                if predict_thr == 0:
-                    # multi-label predict_thr scalar 0 force to 0.5 default
-                    predict_thr = np.atleast_1d(0.5)
-                else:
-                    predict_thr = np.atleast_1d(predict_thr)
-            else:
-                if predict_thr.ndim == 1:
-                    if predict_thr[0] == 0:
-                        # multi-label predict_thr array scalar 0 force to 0.5 default
-                        predict_thr = np.atleast_1d(0.5)
-                    else:
-                        # multi-label predict_thr array set to scalar = array[0]
-                        predict_thr = predict_thr[0]
-    else:
-        # single-label mode, force argmax mode
-        predict_thr = np.atleast_1d(0)
-
-    if predict_thr == -1:
-        thrpr, thrroc, _, _ = calc_optimal_thresholds(truth, predict, timesteps)
-        predict_thr = np.atleast_1d(thrpr)
-        predict_thr = np.maximum(predict_thr, 0.001)  # enforce lower limit
-        predict_thr = np.minimum(predict_thr, 0.999)  # enforce upper limit
-        predict_thr = predict_thr.round(2)
-
-    # Summarize the mixture data
-    num_mixtures = mixdb.num_mixtures
-    total_samples = sum([mixture.samples for mixture in mixdb.mixtures])
-    duration = total_samples / SAMPLE_RATE
-
-    logger.info('')
-    logger.info(f'Mixtures: {num_mixtures}')
-    logger.info(f'Duration: {seconds_to_hms(seconds=duration)}')
-    logger.info(f'truth:    {human_readable_size(truth.nbytes, 1)}')
-    logger.info(f'predict:  {human_readable_size(predict.nbytes, 1)}')
-    if compute_segsnr:
-        logger.info(f'segsnr:   {human_readable_size(segsnr.nbytes, 1)}')
-    if feature:
-        logger.info(f'feature:  {human_readable_size(feature.nbytes, 1)}')
-
-    logger.info(f'Classes: {num_classes}')
-    if mixdb.truth_mutex:
-        logger.info(f'Mode:  Single-label / truth_mutex / softmax')
-    else:
-        logger.info(f'Mode:  Multi-label / Binary')
-
-    mxid_snro = get_mixids_from_snr(mixdb=mixdb)
-    snrlist = list(mxid_snro.keys())
-    snrlist.sort(reverse=True)
-    logger.info(f'Ordered SNRs: {snrlist}\n')
-    predict_thr_info = predict_thr.transpose() if isinstance(predict_thr, np.ndarray) else predict_thr
-    logger.info(f'Prediction Threshold(s): {predict_thr_info}\n')
-
-    # Top-level report over all mixtures
-    macrodf, microdf, wghtdf, mxid_snro = snr_summary(mixdb=mixdb,
-                                                      mixid=':',
-                                                      truth_f=truth,
-                                                      predict=predict,
-                                                      segsnr=segsnr if compute_segsnr else None,
-                                                      predict_thr=predict_thr)
-
-    if num_classes > 1:
-        logger.info(f'Metrics micro-avg per SNR over all {num_mixtures} mixtures:')
-    else:
-        logger.info(f'Metrics per SNR over all {num_mixtures} mixtures:')
-    logger.info(microdf.round(3).to_string())
-    logger.info('')
-    if output_dir:
-        microdf.round(3).to_csv(join(output_dir, 'snr.csv'))
-
-    if mixdb.truth_mutex:
-        macrodf, microdf, wghtdf, mxid_snro = snr_summary(mixdb=mixdb,
-                                                          mixid=':',
-                                                          truth_f=truth[:, 0:-1],
-                                                          predict=predict[:, 0:-1],
-                                                          segsnr=segsnr if compute_segsnr else None,
-                                                          predict_thr=predict_thr)
-
-        logger.info(f'Metrics micro-avg without "Other" class per SNR over all {num_mixtures} mixtures:')
-        logger.info(microdf.round(3).to_string())
-        logger.info('')
-        if output_dir:
-            microdf.round(3).to_csv(join(output_dir, 'snrwo.csv'))
-
-    for snri in snrlist:
-        mxids = mxid_snro[snri]
-        classdf = class_summary(mixdb, mxids, truth, predict, predict_thr)
-        logger.info(f'Metrics per class for SNR {snri} over {len(mxids)} mixtures:')
-        logger.info(classdf.round(3).to_string())
-        logger.info('')
-        if output_dir:
-            classdf.round(3).to_csv(join(output_dir, f'class_snr{snri}.csv'))
-
-
-def main() -> None:
-    from datetime import datetime
-    from os import mkdir
-    from os.path import join
-
-    import h5py
-    from docopt import docopt
-
-    import sonusai
-    from sonusai import SonusAIError
-    from sonusai import create_file_handler
-    from sonusai.utils import read_predict_data
-    from sonusai.utils import trim_docstring
-
-    args = docopt(trim_docstring(__doc__), version=sonusai.__version__, options_first=True)
-
-    verbose = args['--verbose']
-    feature_name = args['--feature']
-    predict_name = args['--predict']
-    predict_threshold = np.array(float(args['--thr']), dtype=np.float32)
-    location = args['LOC']
-
-    mixdb = MixtureDatabase(location)
-
-    # create output directory
-    output_dir = f'evaluate-{datetime.now():%Y%m%d}'
-    try:
-        mkdir(output_dir)
-    except OSError as _:
-        output_dir = f'evaluate-{datetime.now():%Y%m%d-%H%M%S}'
-        try:
-            mkdir(output_dir)
-        except OSError as error:
-            raise SonusAIError(f'Could not create directory, {output_dir}: {error}')
-
-    create_file_handler(join(output_dir, 'evaluate.log'))
-
-    with h5py.File(feature_name, 'r') as f:
-        truth_f = np.array(f['truth_f'])
-        segsnr = np.array(f['segsnr'])
-
-    predict = read_predict_data(predict_name)
-
-    evaluate(mixdb=mixdb,
-             truth=truth_f,
-             segsnr=segsnr,
-             output_dir=output_dir,
-             predict=predict,
-             predict_thr=predict_threshold,
-             verbose=verbose)
-
-    logger.info(f'Wrote results to {output_dir}')
-
-
-if __name__ == '__main__':
-    try:
-        main()
-    except KeyboardInterrupt:
-        logger.info('Canceled due to keyboard interrupt')
-        raise SystemExit(0)

sonusai/keras_onnx.py

@@ -1,86 +0,0 @@
-"""sonusai keras_onnx
-
-usage: keras_onnx [-hvr] (-m MODEL) (-w WEIGHTS) [-b BATCH] [-t TSTEPS] [-o OUTPUT]
-
-options:
-    -h, --help
-    -v, --verbose                   Be verbose.
-    -m MODEL, --model MODEL         Python model file.
-    -w WEIGHTS, --weights WEIGHTS   Keras model weights file.
-    -b BATCH, --batch BATCH         Batch size.
-    -t TSTEPS, --tsteps TSTEPS      Timesteps.
-    -o OUTPUT, --output OUTPUT      Output directory.
-
-Convert a trained Keras model to ONNX.
-
-Inputs:
-    MODEL       A SonusAI Python model file with build and/or hypermodel functions.
-    WEIGHTS     A Keras model weights file (or model file with weights).
-
-Outputs:
-    OUTPUT/     A directory containing:
-                    <MODEL>.onnx        Model file with batch_size and timesteps equal to provided parameters
-                    <MODEL>-b1.onnx     Model file with batch_size=1 and if the timesteps dimension exists it
-                                        is set to 1 (useful for real-time inference applications)
-                    keras_onnx.log
-
-Results are written into subdirectory <MODEL>-<TIMESTAMP> unless OUTPUT is specified.
-
-"""
-from sonusai import logger
-
-
-def main() -> None:
-    from docopt import docopt
-
-    import sonusai
-    from sonusai.utils import trim_docstring
-
-    args = docopt(trim_docstring(__doc__), version=sonusai.__version__, options_first=True)
-
-    verbose = args['--verbose']
-    model_name = args['--model']
-    weight_name = args['--weights']
-    batch_size = args['--batch']
-    timesteps = args['--tsteps']
-    output_dir = args['--output']
-
-    from os import makedirs
-    from os.path import basename
-    from os.path import join
-    from os.path import splitext
-
-    from sonusai import create_file_handler
-    from sonusai import initial_log_messages
-    from sonusai import update_console_handler
-    from sonusai.utils import create_ts_name
-    from sonusai.utils import keras_onnx
-
-    model_tail = basename(model_name)
-    model_root = splitext(model_tail)[0]
-
-    if batch_size is not None:
-        batch_size = int(batch_size)
-
-    if timesteps is not None:
-        timesteps = int(timesteps)
-
-    if output_dir is None:
-        output_dir = create_ts_name(model_root)
-
-    makedirs(output_dir, exist_ok=True)
-
-    # Setup logging file
-    create_file_handler(join(output_dir, 'keras_onnx.log'))
-    update_console_handler(verbose)
-    initial_log_messages('keras_onnx')
-
-    keras_onnx(model_name, weight_name, timesteps, batch_size, output_dir)
-
-
-if __name__ == '__main__':
-    try:
-        main()
-    except KeyboardInterrupt:
-        logger.info('Canceled due to keyboard interrupt')
-        exit()

sonusai/keras_predict.py

@@ -1,231 +0,0 @@
-"""sonusai keras_predict
-
-usage: keras_predict [-hvr] [-i MIXID] (-m MODEL) (-w KMODEL) [-b BATCH] [-t TSTEPS] INPUT ...
-
-options:
-    -h, --help
-    -v, --verbose                   Be verbose.
-    -i MIXID, --mixid MIXID         Mixture ID(s) to generate if input is a mixture database. [default: *].
-    -m MODEL, --model MODEL         Python model file.
-    -w KMODEL, --weights KMODEL     Keras model weights file.
-    -b BATCH, --batch BATCH         Batch size.
-    -t TSTEPS, --tsteps TSTEPS      Timesteps.
-    -r, --reset                     Reset model between each file.
-
-Run prediction on a trained Keras model defined by a SonusAI Keras Python model file using SonusAI genft or WAV data.
-
-Inputs:
-    MODEL       A SonusAI Python model file with build and/or hypermodel functions.
-    KMODEL      A Keras model weights file (or model file with weights).
-    INPUT       The input data must be one of the following:
-                * Single WAV file or glob of WAV files
-                  Using the given model, generate feature data and run prediction. A model file must be
-                  provided. The MIXID is ignored.
-
-                * directory
-                  Using the given SonusAI mixture database directory, generate feature and truth data if not found.
-                  Run prediction. The MIXID is required.
-
-Outputs the following to kpredict-<TIMESTAMP> directory:
-    <id>.h5
-        dataset:    predict
-    keras_predict.log
-
-"""
-from typing import Any
-
-from sonusai import logger
-from sonusai.mixture import Feature
-from sonusai.mixture import Predict
-
-
-def main() -> None:
-    from docopt import docopt
-
-    import sonusai
-    from sonusai.utils import trim_docstring
-
-    args = docopt(trim_docstring(__doc__), version=sonusai.__version__, options_first=True)
-
-    verbose = args['--verbose']
-    mixids = args['--mixid']
-    model_name = args['--model']
-    weights_name = args['--weights']
-    batch_size = args['--batch']
-    timesteps = args['--tsteps']
-    reset = args['--reset']
-    input_name = args['INPUT']
-
-    from os import makedirs
-    from os.path import basename
-    from os.path import isdir
-    from os.path import isfile
-    from os.path import join
-    from os.path import splitext
-
-    import h5py
-    import keras_tuner as kt
-    import tensorflow as tf
-    from keras import backend as kb
-
-    from sonusai import create_file_handler
-    from sonusai import initial_log_messages
-    from sonusai import update_console_handler
-    from sonusai.data_generator import KerasFromH5
-    from sonusai.mixture import MixtureDatabase
-    from sonusai.mixture import get_feature_from_audio
-    from sonusai.mixture import read_audio
-    from sonusai.utils import create_ts_name
-    from sonusai.utils import get_frames_per_batch
-    from sonusai.utils import import_and_check_keras_model
-    from sonusai.utils import reshape_outputs
-
-    if batch_size is not None:
-        batch_size = int(batch_size)
-
-    if timesteps is not None:
-        timesteps = int(timesteps)
-
-    output_dir = create_ts_name('kpredict')
-    makedirs(output_dir, exist_ok=True)
-
-    # Setup logging file
-    create_file_handler(join(output_dir, 'keras_predict.log'))
-    update_console_handler(verbose)
-    initial_log_messages('keras_predict')
-
-    logger.info(f'tensorflow    {tf.__version__}')
-    logger.info(f'keras         {tf.keras.__version__}')
-    logger.info('')
-
-    hypermodel = import_and_check_keras_model(model_name=model_name,
-                                              weights_name=weights_name,
-                                              timesteps=timesteps,
-                                              batch_size=batch_size)
-    built_model = hypermodel.build_model(kt.HyperParameters())
-
-    frames_per_batch = get_frames_per_batch(hypermodel.batch_size, hypermodel.timesteps)
-
-    kb.clear_session()
-    logger.info('')
-    built_model.summary(print_fn=logger.info)
-    logger.info('')
-    logger.info(f'feature       {hypermodel.feature}')
-    logger.info(f'num_classes   {hypermodel.num_classes}')
-    logger.info(f'batch_size    {hypermodel.batch_size}')
-    logger.info(f'timesteps     {hypermodel.timesteps}')
-    logger.info(f'flatten       {hypermodel.flatten}')
-    logger.info(f'add1ch        {hypermodel.add1ch}')
-    logger.info(f'truth_mutex   {hypermodel.truth_mutex}')
-    logger.info(f'input_shape   {hypermodel.input_shape}')
-    logger.info('')
-
-    logger.info(f'Loading weights from {weights_name}')
-    built_model.load_weights(weights_name)
-
-    logger.info('')
-    if len(input_name) == 1 and isdir(input_name[0]):
-        input_name = input_name[0]
-        logger.info(f'Load mixture database from {input_name}')
-        mixdb = MixtureDatabase(input_name)
-
-        if mixdb.feature != hypermodel.feature:
-            logger.exception(f'Feature in mixture database does not match feature in model')
-            raise SystemExit(1)
-
-        mixids = mixdb.mixids_to_list(mixids)
-        if reset:
-            # reset mode cycles through each file one at a time
-            for mixid in mixids:
-                feature, _ = mixdb.mixture_ft(mixid)
-
-                feature, predict = _pad_and_predict(hypermodel=hypermodel,
-                                                    built_model=built_model,
-                                                    feature=feature,
-                                                    frames_per_batch=frames_per_batch)
-
-                output_name = join(output_dir, mixdb.mixtures[mixid].name)
-                with h5py.File(output_name, 'a') as f:
-                    if 'predict' in f:
-                        del f['predict']
-                    f.create_dataset(name='predict', data=predict)
-        else:
-            # Run all data at once using a data generator
-            feature = KerasFromH5(mixdb=mixdb,
-                                  mixids=mixids,
-                                  batch_size=hypermodel.batch_size,
-                                  timesteps=hypermodel.timesteps,
-                                  flatten=hypermodel.flatten,
-                                  add1ch=hypermodel.add1ch)
-
-            predict = built_model.predict(feature, batch_size=hypermodel.batch_size, verbose=1)
-            predict, _ = reshape_outputs(predict=predict, timesteps=hypermodel.timesteps)
-
-            # Write data to separate files
-            for idx, mixid in enumerate(mixids):
-                output_name = join(output_dir, mixdb.mixtures[mixid].name)
-                with h5py.File(output_name, 'a') as f:
-                    if 'predict' in f:
-                        del f['predict']
-                    f.create_dataset('predict', data=predict[feature.file_indices[idx]])
-
-        logger.info(f'Saved results to {output_dir}')
-        return
-
-    if not all(isfile(file) and splitext(file)[1] == '.wav' for file in input_name):
-        logger.exception(f'Do not know how to process input from {input_name}')
-        raise SystemExit(1)
-
-    logger.info(f'Run prediction on {len(input_name):,} WAV files')
-    for file in input_name:
-        # Convert WAV to feature data
-        audio = read_audio(file)
-        feature = get_feature_from_audio(audio=audio, feature=hypermodel.feature)
-
-        feature, predict = _pad_and_predict(hypermodel=hypermodel,
-                                            built_model=built_model,
-                                            feature=feature,
-                                            frames_per_batch=frames_per_batch)
-
-        output_name = join(output_dir, splitext(basename(file))[0] + '.h5')
-        with h5py.File(output_name, 'a') as f:
-            if 'feature' in f:
-                del f['feature']
-            f.create_dataset(name='feature', data=feature)
-
-            if 'predict' in f:
-                del f['predict']
-            f.create_dataset(name='predict', data=predict)
-
-    logger.info(f'Saved results to {output_dir}')
-
-
-def _pad_and_predict(hypermodel: Any,
-                     built_model: Any,
-                     feature: Feature,
-                     frames_per_batch: int) -> tuple[Feature, Predict]:
-    import numpy as np
-
-    from sonusai.utils import reshape_inputs
-    from sonusai.utils import reshape_outputs
-
-    frames = feature.shape[0]
-    padding = frames_per_batch - frames % frames_per_batch
-    feature = np.pad(array=feature, pad_width=((0, padding), (0, 0), (0, 0)))
-    feature, _ = reshape_inputs(feature=feature,
-                                batch_size=hypermodel.batch_size,
-                                timesteps=hypermodel.timesteps,
-                                flatten=hypermodel.flatten,
-                                add1ch=hypermodel.add1ch)
-    predict = built_model.predict(feature, batch_size=hypermodel.batch_size, verbose=1)
-    predict, _ = reshape_outputs(predict=predict, timesteps=hypermodel.timesteps)
-    predict = predict[:frames, :]
-    return feature, predict
-
-
-if __name__ == '__main__':
-    try:
-        main()
-    except KeyboardInterrupt:
-        logger.info('Canceled due to keyboard interrupt')
-        exit()