sonusai 0.15.9__py3-none-any.whl → 0.16.1__py3-none-any.whl

sonusai/keras_train.py

@@ -1,334 +0,0 @@
-"""sonusai keras_train
-
-usage: keras_train [-hgv] (-m MODEL) (-l VLOC) [-w KMODEL] [-e EPOCHS] [-b BATCH] [-t TSTEPS] [-p ESP] TLOC
-
-options:
-    -h, --help
-    -v, --verbose                   Be verbose.
-    -m MODEL, --model MODEL         Model Python file with build and/or hypermodel functions.
-    -l VLOC, --vloc VLOC            Location of SonusAI mixture database to use for validation.
-    -w KMODEL, --weights KMODEL     Keras model weights file.
-    -e EPOCHS, --epochs EPOCHS      Number of epochs to use in training. [default: 8].
-    -b BATCH, --batch BATCH         Batch size.
-    -t TSTEPS, --tsteps TSTEPS      Timesteps.
-    -p ESP, --patience ESP          Early stopping patience.
-    -g, --loss-batch-log            Enable per-batch loss log. [default: False]
-
-Use Keras to train a model defined by a Python definition file and SonusAI genft data.
-
-Inputs:
-    TLOC    A SonusAI mixture database directory to use for training data.
-    VLOC    A SonusAI mixture database directory to use for validation data.
-
-Results are written into subdirectory <MODEL>-<TIMESTAMP>.
-Per-batch loss history, if enabled, is written to <basename>-history-lossb.npy
-
-"""
-import tensorflow as tf
-
-from sonusai import logger
-
-
-class LossBatchHistory(tf.keras.callbacks.Callback):
-    def __init__(self):
-        super().__init__()
-        self.history = None
-
-    def on_train_begin(self, logs=None):
-        self.history = {'loss': []}
-
-    def on_batch_end(self, batch, logs=None):
-        if logs is None:
-            logs = {}
-        self.history['loss'].append(logs.get('loss'))
-
-
-class SonusAIModelCheckpoint(tf.keras.callbacks.ModelCheckpoint):
-    def __init__(self,
-                 filepath,
-                 monitor: str = "val_loss",
-                 verbose: int = 0,
-                 save_best_only: bool = False,
-                 save_weights_only: bool = False,
-                 mode: str = "auto",
-                 save_freq="epoch",
-                 options=None,
-                 initial_value_threshold=None,
-                 **kwargs):
-        super().__init__(filepath,
-                         monitor,
-                         verbose,
-                         save_best_only,
-                         save_weights_only,
-                         mode,
-                         save_freq,
-                         options,
-                         initial_value_threshold,
-                         **kwargs)
-        self.feature = kwargs.get('feature', None)
-        self.num_classes = kwargs.get('num_classes', None)
-
-    def _save_model(self, epoch, batch, logs):
-        import h5py
-
-        super()._save_model(epoch, batch, logs)
-
-        with h5py.File(self.filepath, 'a') as f:
-            if self.feature is not None:
-                f.attrs['sonusai_feature'] = self.feature
-            if self.num_classes is not None:
-                f.attrs['sonusai_num_classes'] = str(self.num_classes)
-
-
-def main() -> None:
-    from docopt import docopt
-
-    import sonusai
-    from sonusai.utils import trim_docstring
-
-    args = docopt(trim_docstring(__doc__), version=sonusai.__version__, options_first=True)
-
-    verbose = args['--verbose']
-    model_name = args['--model']
-    weights_name = args['--weights']
-    v_name = args['--vloc']
-    epochs = int(args['--epochs'])
-    batch_size = args['--batch']
-    timesteps = args['--tsteps']
-    esp = args['--patience']
-    loss_batch_log = args['--loss-batch-log']
-    t_name = args['TLOC']
-
-    import warnings
-    from os import makedirs
-    from os import walk
-    from os.path import basename
-    from os.path import join
-    from os.path import splitext
-
-    import h5py
-    import keras_tuner as kt
-    import numpy as np
-
-    with warnings.catch_warnings():
-        warnings.simplefilter('ignore')
-        from keras import backend as kb
-        from keras.callbacks import EarlyStopping
-
-    from sonusai import create_file_handler
-    from sonusai import initial_log_messages
-    from sonusai import update_console_handler
-    from sonusai.data_generator import KerasFromH5
-    from sonusai.mixture import MixtureDatabase
-    from sonusai.utils import check_keras_overrides
-    from sonusai.utils import create_ts_name
-    from sonusai.utils import import_keras_model
-    from sonusai.utils import stratified_shuffle_split_mixid
-    from sonusai.utils import reshape_outputs
-    from sonusai.utils import get_frames_per_batch
-
-    model_base = basename(model_name)
-    model_root = splitext(model_base)[0]
-
-    if batch_size is not None:
-        batch_size = int(batch_size)
-
-    if timesteps is not None:
-        timesteps = int(timesteps)
-
-    output_dir = create_ts_name(model_root)
-    makedirs(output_dir, exist_ok=True)
-    base_name = join(output_dir, model_root)
-
-    # Setup logging file
-    create_file_handler(join(output_dir, 'keras_train.log'))
-    update_console_handler(verbose)
-    initial_log_messages('keras_train')
-
-    logger.info(f'tensorflow    {tf.__version__}')
-    logger.info(f'keras         {tf.keras.__version__}')
-    logger.info('')
-
-    t_mixdb = MixtureDatabase(t_name)
-    logger.info(f'Training: found {len(t_mixdb.mixtures)} mixtures with {t_mixdb.num_classes} classes from {t_name}')
-
-    v_mixdb = MixtureDatabase(v_name)
-    logger.info(f'Validation: found {len(v_mixdb.mixtures)} mixtures with {v_mixdb.num_classes} classes from {v_name}')
-
-    # Import model definition file
-    logger.info(f'Importing {model_base}')
-    model = import_keras_model(model_name)
-
-    # Check overrides
-    timesteps = check_keras_overrides(model, t_mixdb.feature, t_mixdb.num_classes, timesteps, batch_size)
-    # Calculate batches per epoch, use ceiling as last batch is zero extended
-    frames_per_batch = get_frames_per_batch(batch_size, timesteps)
-    batches_per_epoch = int(np.ceil(t_mixdb.total_feature_frames('*') / frames_per_batch))
-
-    logger.info('Building and compiling model')
-    try:
-        hypermodel = model.MyHyperModel(feature=t_mixdb.feature,
-                                        num_classes=t_mixdb.num_classes,
-                                        timesteps=timesteps,
-                                        batch_size=batch_size)
-        built_model = hypermodel.build_model(kt.HyperParameters())
-        built_model = hypermodel.compile_default(built_model, batches_per_epoch)
-    except Exception as e:
-        logger.exception(f'Error: build_model() in {model_base} failed: {e}')
-        raise SystemExit(1)
-
-    kb.clear_session()
-    logger.info('')
-    built_model.summary(print_fn=logger.info)
-    logger.info('')
-    logger.info(f'feature       {hypermodel.feature}')
-    logger.info(f'num_classes   {hypermodel.num_classes}')
-    logger.info(f'batch_size    {hypermodel.batch_size}')
-    logger.info(f'timesteps     {hypermodel.timesteps}')
-    logger.info(f'flatten       {hypermodel.flatten}')
-    logger.info(f'add1ch        {hypermodel.add1ch}')
-    logger.info(f'truth_mutex   {hypermodel.truth_mutex}')
-    logger.info(f'lossf         {hypermodel.lossf}')
-    logger.info(f'input_shape   {hypermodel.input_shape}')
-    logger.info(f'optimizer     {built_model.optimizer.get_config()}')
-    logger.info('')
-
-    t_mixid = t_mixdb.mixids_to_list()
-    v_mixid = v_mixdb.mixids_to_list()
-
-    stratify = False
-    if stratify:
-        logger.info(f'Stratifying training data')
-        t_mixid, _, _, _ = stratified_shuffle_split_mixid(t_mixdb, vsplit=0)
-
-    # Use SonusAI DataGenerator to create validation feature/truth on the fly
-    v_datagen = KerasFromH5(mixdb=v_mixdb,
-                            mixids=v_mixid,
-                            batch_size=hypermodel.batch_size,
-                            timesteps=hypermodel.timesteps,
-                            flatten=hypermodel.flatten,
-                            add1ch=hypermodel.add1ch,
-                            shuffle=False)
-
-    # Prepare class weighting
-    # class_count = np.ceil(np.array(get_class_count_from_mixids(t_mixdb, t_mixid)) / t_mixdb.feature_step_samples)
-    # if t_mixdb.truth_mutex:
-    #     other_weight = 16.0
-    #     logger.info(f'Detected single-label mode (truth_mutex); setting other weight to {other_weight}')
-    #     class_count[-1] = class_count[-1] / other_weight
-
-    # Use SonusAI DataGenerator to create training feature/truth on the fly
-    t_datagen = KerasFromH5(mixdb=t_mixdb,
-                            mixids=t_mixid,
-                            batch_size=hypermodel.batch_size,
-                            timesteps=hypermodel.timesteps,
-                            flatten=hypermodel.flatten,
-                            add1ch=hypermodel.add1ch,
-                            shuffle=True)
-
-    # TODO: If hypermodel.es exists, then use it; otherwise use default here
-    if esp is None:
-        es = EarlyStopping(monitor='val_loss',
-                           mode='min',
-                           verbose=1,
-                           patience=8)
-    else:
-        es = EarlyStopping(monitor='val_loss',
-                           mode='min',
-                           verbose=1,
-                           patience=int(esp))
-
-    ckpt_callback = SonusAIModelCheckpoint(filepath=base_name + '-ckpt-weights.h5',
-                                           save_weights_only=True,
-                                           monitor='val_loss',
-                                           mode='min',
-                                           save_best_only=True,
-                                           feature=hypermodel.feature,
-                                           num_classes=hypermodel.num_classes)
-
-    csv_logger = tf.keras.callbacks.CSVLogger(base_name + '-history.csv')
-    callbacks = [es, ckpt_callback, csv_logger]
-    # loss_batch_log = True
-    loss_batchlogger = None
-    if loss_batch_log is True:
-        loss_batchlogger = LossBatchHistory()
-        callbacks.append(loss_batchlogger)
-        logger.info(f'Adding per batch loss logging to training')
-
-    if weights_name is not None:
-        logger.info(f'Loading weights from {weights_name}')
-        built_model.load_weights(weights_name)
-
-    logger.info('')
-    logger.info(f'Training with no class weighting and early stopping patience = {es.patience}')
-    logger.info(f'  training mixtures    {len(t_mixid)}')
-    logger.info(f'  validation mixtures  {len(v_mixid)}')
-    logger.info('')
-
-    history = built_model.fit(t_datagen,
-                              batch_size=hypermodel.batch_size,
-                              epochs=epochs,
-                              validation_data=v_datagen,
-                              shuffle=False,
-                              callbacks=callbacks)
-
-    # Save history into numpy file
-    history_name = base_name + '-history'
-    np.save(history_name, history.history)
-    # Note: Reload with history=np.load(history_name, allow_pickle='TRUE').item()
-    logger.info(f'Saved training history to numpy file {history_name}.npy')
-    if loss_batch_log is True:
-        his_batch_loss_name = base_name + '-history-lossb.npy'
-        np.save(his_batch_loss_name, loss_batchlogger.history)
-        logger.info(f'Saved per-batch loss history to numpy file {his_batch_loss_name}')
-
-    # Find checkpoint file and load weights for prediction and model save
-    checkpoint_name = None
-    for path, dirs, files in walk(output_dir):
-        for file in files:
-            if "ckpt" in file:
-                checkpoint_name = file
-
-    if checkpoint_name is not None:
-        logger.info('Using best checkpoint for prediction and model exports')
-        built_model.load_weights(join(output_dir, checkpoint_name))
-    else:
-        logger.info('Using last epoch for prediction and model exports')
-
-    # save for later model export(s)
-    weight_name = base_name + '.h5'
-    built_model.save(weight_name)
-    with h5py.File(weight_name, 'a') as f:
-        f.attrs['sonusai_feature'] = hypermodel.feature
-        f.attrs['sonusai_num_classes'] = str(hypermodel.num_classes)
-    logger.info(f'Saved trained model to {weight_name}')
-
-    # Compute prediction metrics on validation data using the best checkpoint
-    v_predict = built_model.predict(v_datagen, batch_size=hypermodel.batch_size, verbose=1)
-    v_predict, _ = reshape_outputs(predict=v_predict, timesteps=hypermodel.timesteps)
-
-    # Write data to separate files
-    v_predict_dir = base_name + '-valpredict'
-    makedirs(v_predict_dir, exist_ok=True)
-    for idx, mixid in enumerate(v_mixid):
-        output_name = join(v_predict_dir, v_mixdb.mixtures[mixid].name)
-        indices = v_datagen.file_indices[idx]
-        frames = indices.stop - indices.start
-        data = v_predict[indices]
-        # The predict operation may produce less data due to timesteps and batches may not dividing evenly
-        # Only write data if it exists
-        if data.shape[0] == frames:
-            with h5py.File(output_name, 'a') as f:
-                if 'predict' in f:
-                    del f['predict']
-                f.create_dataset('predict', data=data)
-
-    logger.info(f'Wrote validation predict data to {v_predict_dir}')
-
-
-if __name__ == '__main__':
-    try:
-        main()
-    except KeyboardInterrupt:
-        logger.info('Canceled due to keyboard interrupt')
-        exit()

sonusai/torchl_onnx.py

@@ -1,216 +0,0 @@
-"""sonusai torchl_onnx
-
-usage: torchl_onnx [-hv] [-b BATCH] [-t TSTEPS] [-o OUTPUT] MODEL CKPT
-
-options:
-    -h, --help
-    -v, --verbose                   Be verbose
-    -b BATCH, --batch BATCH         Batch size [default: 1]
-    -t TSTEPS, --tsteps TSTEPS      Timesteps [default: 1]
-    -o OUTPUT, --output OUTPUT      Output directory.
-
-Convert a trained Pytorch Lightning model to ONNX.  The model is specified as an
-sctl_*.py model file (sctl: sonusai custom torch lightning) and a checkpoint file
-for loading weights.
-
-Inputs:
-    MODEL       SonusAI Python custom model file.
-    CKPT        A Pytorch Lightning checkpoint file
-    BATCH       Batch size used in onnx conversion, overrides value in model ckpt. Defaults to 1.
-    TSTEPS      Timestep dimension size using in onnx conversion, overrides value in model ckpt if
-                the model has a timestep dimension.  Else it is ignored.
-
-Outputs:
-    OUTPUT/     A directory containing:
-                    <CKPT>.onnx        Model file with batch_size and timesteps equal to provided parameters
-                    <CKPT>-b1.onnx     Model file with batch_size=1 and if the timesteps dimension exists it
-                                       is set to 1 (useful for real-time inference applications)
-                    torchl_onnx.log
-
-Results are written into subdirectory <MODEL>-<TIMESTAMP> unless OUTPUT is specified.
-
-"""
-from sonusai import logger
-
-
-def main() -> None:
-    from docopt import docopt
-
-    import sonusai
-    from sonusai.utils import trim_docstring
-
-    args = docopt(trim_docstring(__doc__), version=sonusai.__version__, options_first=True)
-
-    verbose = args['--verbose']
-    batch_size = args['--batch']
-    timesteps = args['--tsteps']
-    model_path = args['MODEL']
-    ckpt_path = args['CKPT']
-    output_dir = args['--output']
-
-    from os import makedirs
-    from os.path import basename, splitext
-    from sonusai.utils import import_keras_model
-
-    # Import model definition file first to check
-    model_base = basename(model_path)
-    model_root = splitext(model_base)[0]
-    logger.info(f'Importing model from {model_base}')
-    try:
-        litemodule = import_keras_model(model_path)  # note works for pytorch lightning as well as keras
-    except Exception as e:
-        logger.exception(f'Error: could not import model from {model_path}: {e}')
-        raise SystemExit(1)
-
-    # Load checkpoint first to get hparams if available
-    from torch import load as load
-    ckpt_base = basename(ckpt_path)
-    ckpt_root = splitext(ckpt_base)[0]
-    logger.info(f'Loading checkpoint from {ckpt_base}')
-    try:
-        checkpoint = load(ckpt_path, map_location=lambda storage, loc: storage)
-    except Exception as e:
-        logger.exception(f'Error: could not load checkpoint from {ckpt_path}: {e}')
-        raise SystemExit(1)
-
-    from os.path import join, isdir, dirname, exists
-    from sonusai import create_file_handler
-    from sonusai import initial_log_messages
-    from sonusai import update_console_handler
-    from torch import randn
-    from sonusai.utils import create_ts_name
-
-    from sonusai.utils import create_ts_name
-    from torchinfo import summary
-
-    if batch_size is not None:
-        batch_size = int(batch_size)
-    if batch_size != 1:
-        batch_size = 1
-        logger.info(f'For now prediction only supports batch_size = 1, forcing it to 1 now')
-
-    if timesteps is not None:
-        timesteps = int(timesteps)
-
-    if output_dir is None:
-        output_dir = dirname(ckpt_path)
-    else:
-        if not isdir(output_dir):
-            makedirs(output_dir, exist_ok=True)
-
-    ofname = join(output_dir, ckpt_root + '.onnx')
-    # First try, then add date
-    if exists(ofname):
-        # add hour-min-sec if necessary
-        from datetime import datetime
-        ts = datetime.now()
-        ofname = join(output_dir, ckpt_root + '-' + ts.strftime('%Y%m%d') + '.onnx')
-    ofname_root = splitext(ofname)[0]
-
-    # Setup logging file
-    create_file_handler(ofname_root + '-onnx.log')
-    update_console_handler(verbose)
-    initial_log_messages('torchl_onnx')
-    logger.info(f'Imported model from {model_base}')
-    logger.info(f'Loaded checkpoint from {ckpt_base}')
-
-    if 'hyper_parameters' in checkpoint:
-        hparams = checkpoint['hyper_parameters']
-        logger.info(f'Found hyper-params on checkpoint named {checkpoint["hparams_name"]} '
-                    f'with {len(hparams)} total hparams.')
-        if batch_size is not None and hparams['batch_size'] != batch_size:
-            if batch_size != 1:
-                batch_size = 1
-                logger.info(f'For now prediction only supports batch_size = 1, forcing it to 1 now')
-            logger.info(f'Overriding batch_size: default = {hparams["batch_size"]}; specified = {batch_size}.')
-            hparams["batch_size"] = batch_size
-
-        if timesteps is not None:
-            if hparams['timesteps'] == 0 and timesteps != 0:
-                logger.warning(f'Model does not contain timesteps; ignoring override.')
-                timesteps = 0
-
-            if hparams['timesteps'] != 0 and timesteps == 0:
-                logger.warning(f'Model contains timesteps; ignoring override of 0, using model default.')
-                timesteps = hparams['timesteps']
-
-            if hparams['timesteps'] != timesteps:
-                logger.info(f'Overriding timesteps: default = {hparams["timesteps"]}; specified = {timesteps}.')
-                hparams['timesteps'] = timesteps
-
-        logger.info(f'Building model with hparams and batch_size={batch_size}, timesteps={timesteps}')
-        try:
-            model = litemodule.MyHyperModel(**hparams)  # use hparams
-            # litemodule.MyHyperModel.load_from_checkpoint(ckpt_name, **hparams)
-        except Exception as e:
-            logger.exception(f'Error: model build (MyHyperModel) in {model_base} failed: {e}')
-            raise SystemExit(1)
-    else:
-        logger.info(f'Warning: found checkpoint with no hyper-parameters, building model with defaults')
-        try:
-            tmp = litemodule.MyHyperModel()  # use default hparams
-        except Exception as e:
-            logger.exception(f'Error: model build (MyHyperModel) in {model_base} failed: {e}')
-            raise SystemExit(1)
-
-        if batch_size is not None:
-            if tmp.batch_size != batch_size:
-                logger.info(f'Overriding batch_size: default = {tmp.batch_size}; specified = {batch_size}.')
-        else:
-            batch_size = tmp.batch_size  # inherit
-
-        if timesteps is not None:
-            if tmp.timesteps == 0 and timesteps != 0:
-                logger.warning(f'Model does not contain timesteps; ignoring override.')
-                timesteps = 0
-
-            if tmp.timesteps != 0 and timesteps == 0:
-                logger.warning(f'Model contains timesteps; ignoring override.')
-                timesteps = tmp.timesteps
-
-            if tmp.timesteps != timesteps:
-                logger.info(f'Overriding timesteps: default = {tmp.timesteps}; specified = {timesteps}.')
-        else:
-            timesteps = tmp.timesteps
-
-        logger.info(f'Building model with default hparams and batch_size= {batch_size}, timesteps={timesteps}')
-        model = litemodule.MyHyperModel(timesteps=timesteps, batch_size=batch_size)
-
-    logger.info('')
-    # logger.info(summary(model))
-    # from lightning.pytorch import Trainer
-    # from lightning.pytorch.callbacks import ModelSummary
-    # trainer = Trainer(callbacks=[ModelSummary(max_depth=2)])
-    # logger.info(trainer.summarize())
-    logger.info('')
-    logger.info(f'feature       {model.hparams.feature}')
-    logger.info(f'num_classes   {model.num_classes}')
-    logger.info(f'batch_size    {model.hparams.batch_size}')
-    logger.info(f'timesteps     {model.hparams.timesteps}')
-    logger.info(f'flatten       {model.flatten}')
-    logger.info(f'add1ch        {model.add1ch}')
-    logger.info(f'truth_mutex   {model.truth_mutex}')
-    logger.info(f'input_shape   {model.input_shape}')
-    logger.info('')
-    logger.info(f'Loading weights from {ckpt_base}')
-    # model = model.load_from_checkpoint(ckpt_path)     # weights only, has problems - needs investigation
-    model.load_state_dict(checkpoint["state_dict"])
-    model.eval()
-    insample_shape = model.input_shape
-    insample_shape.insert(0, batch_size)
-    input_sample = randn(insample_shape)
-    logger.info(f'Creating onnx model ...')
-    for m in model.modules():
-        if 'instancenorm' in m.__class__.__name__.lower():
-            logger.info(f'Forcing train=false for instancenorm instance {m}, {m.__class__.__name__.lower()}')
-            m.train(False)
-            # m.track_running_stats=True  # has problems
-    model.to_onnx(file_path=ofname, input_sample=input_sample, export_params=True)
-
-
-if __name__ == '__main__':
-    try:
-        main()
-    except KeyboardInterrupt:
-        logger.info('Canceled due to keyboard interrupt')
-        exit()