sonusai 0.19.9__py3-none-any.whl → 0.20.2__py3-none-any.whl

sonusai/data/genmixdb.yml

@@ -23,7 +23,8 @@ truth_configs: { }
 
 asr_manifest: [ ]
 
-target_augmentations: [ ]
+target_augmentations:
+  - pre:
 
 class_balancing_augmentation:
   normalize: -3.5
@@ -39,7 +40,8 @@ noises:
   - "${default_noise}"
 
 noise_augmentations:
-  - normalize: -3.5
+  - pre:
+      normalize: -3.5
 
 snrs:
   - 99

sonusai/doc/doc.py

@@ -329,6 +329,20 @@ See 'augmentations' for details on augmentation rules.
     # fmt: on
 
 
+def doc_target_distortions() -> str:
+    import yaml
+
+    default = f"\nDefault value:\n\n{yaml.dump(get_default_config()['target_distortions'])}"
+    # fmt: off
+    return """
+'target_distortions' is a mixture database configuration parameter that
+specifies a list of distortion rules to use for each target.
+
+See 'augmentations' for details on distortion rules.
+""" + default
+    # fmt: on
+
+
 def doc_noises() -> str:
     default = f"\nDefault value: {get_default_config()['class_balancing']}"
     # fmt: off

sonusai/genft.py

@@ -109,7 +109,7 @@ def _genft_kernel(
             write_cached_data(mixdb.location, "mixture", mixdb.mixture(m_id).name, [("segsnr", segsnr)])
 
     if write:
-        write_mixture_metadata(mixdb, m_id)
+        write_mixture_metadata(mixdb, m_id=m_id)
 
     return result
 

sonusai/genmetrics.py

@@ -1,6 +1,6 @@
 """sonusai genmetrics
 
-usage: genmetrics [-hvsd] [-i MIXID] [-n INCLUDE] [-x EXCLUDE] LOC
+usage: genmetrics [-hvusd] [-i MIXID] [-n INCLUDE] [-x EXCLUDE] LOC
 
 options:
     -h, --help
@@ -8,6 +8,7 @@ options:
     -i MIXID, --mixid MIXID         Mixture ID(s) to generate. [default: *].
     -n INCLUDE, --include INCLUDE   Metrics to include. [default: all]
     -x EXCLUDE, --exclude EXCLUDE   Metrics to exclude. [default: none]
+    -u, --update                    Update metrics (do not regenerate existing metrics).
     -s, --supported                 Show list of supported metrics.
     -d, --dryrun                    Show list of metrics that will be generated and exit.
 
@@ -60,16 +61,15 @@ def signal_handler(_sig, _frame):
 signal.signal(signal.SIGINT, signal_handler)
 
 
-def _process_mixture(mixid: int, location: str, metrics: list[str]) -> None:
+def _process_mixture(mixid: int, location: str, metrics: list[str], update: bool = False) -> set[str]:
     from sonusai.mixture import MixtureDatabase
     from sonusai.mixture import write_cached_data
 
     mixdb = MixtureDatabase(location)
+    results = mixdb.mixture_metrics(m_id=mixid, metrics=metrics, force=not update)
+    write_cached_data(mixdb.location, "mixture", mixdb.mixture(mixid).name, list(results.items()))
 
-    values = mixdb.mixture_metrics(m_id=mixid, metrics=metrics, force=True)
-    write_data = list(zip(metrics, values, strict=False))
-
-    write_cached_data(mixdb.location, "mixture", mixdb.mixture(mixid).name, write_data)
+    return set(results.keys())
 
 
 def main() -> None:
@@ -85,6 +85,7 @@ def main() -> None:
     mixids = args["--mixid"]
     includes = {x.strip() for x in args["--include"].replace(" ", ",").lower().split(",") if x != ""}
     excludes = {x.strip() for x in args["--exclude"].replace(" ", ",").lower().split(",") if x != ""}
+    update = args["--update"]
     show_supported = args["--supported"]
     dryrun = args["--dryrun"]
     location = args["LOC"]
@@ -141,20 +142,14 @@ def main() -> None:
 
     requested = included_metrics - excluded_metrics
 
-    # Check for metrics dependencies and cache dependencies even if not explicitly requested.
-    dependencies: set[str] = set()
-    for metric in requested:
-        if metric.startswith("mxwer"):
-            dependencies.add("mxasr." + metric[6:])
-            dependencies.add("tasr." + metric[6:])
-
-    metrics = sorted(requested | dependencies)
+    metrics = sorted(requested)
 
     if len(metrics) == 0:
         logger.warning("No metrics were requested")
         sys.exit(1)
 
-    logger.info(f"Generating metrics: {', '.join(metrics)}")
+    logger.info("Generating metrics:")
+    logger.info(f"{', '.join(metrics)}")
     if dryrun:
         sys.exit(0)
 
@@ -163,14 +158,16 @@ def main() -> None:
     logger.info(f"Found {len(mixids):,} mixtures to process")
 
     progress = track(total=len(mixids), desc="genmetrics")
-    par_track(
-        partial(_process_mixture, location=location, metrics=metrics),
+    results = par_track(
+        partial(_process_mixture, location=location, metrics=metrics, update=update),
         mixids,
         progress=progress,
     )
     progress.close()
 
-    logger.info(f"Wrote metrics for {len(mixids)} mixtures to {location}")
+    written_metrics = sorted(set().union(*results))
+    logger.info(f"Wrote metrics for {len(mixids)} mixtures to {location}:")
+    logger.info(f"{', '.join(written_metrics)}")
     logger.info("")
 
     end_time = time.monotonic()

sonusai/genmix.py

@@ -139,7 +139,7 @@ def _genmix_kernel(
     result.mixture = mixture
     if write:
         write_cached_data(mixdb.location, "mixture", mixdb.mixture(m_id).name, [("mixture", mixture)])
-        write_mixture_metadata(mixdb, m_id)
+        write_mixture_metadata(mixdb, m_id=m_id)
 
     return result
 

sonusai/genmixdb.py

@@ -1,13 +1,11 @@
 """sonusai genmixdb
 
-usage: genmixdb [-hvmfsdjn] LOC
+usage: genmixdb [-hvmdjn] LOC
 
 options:
     -h, --help
     -v, --verbose   Be verbose.
     -m, --mix       ave mixture data. [default: False].
-    -f, --ft        Save feature/truth_f data. [default: False].
-    -s, --segsnr    Save segsnr data. [default: False].
     -d, --dryrun    Perform a dry run showing the processed config. [default: False].
     -j, --json      Save JSON version of database. [default: False].
     -n, --nopar     Do not run in parallel. [default: False].
@@ -116,6 +114,9 @@ will find all .wav files in the specified directories and process them as target
 
 import signal
 
+from sonusai.mixture import Mixture
+from sonusai.mixture import MixtureDatabase
+
 
 def signal_handler(_sig, _frame):
     import sys
@@ -132,8 +133,6 @@ signal.signal(signal.SIGINT, signal_handler)
 def genmixdb(
     location: str,
     save_mix: bool = False,
-    save_ft: bool = False,
-    save_segsnr: bool = False,
     logging: bool = True,
     show_progress: bool = False,
     test: bool = False,
@@ -151,6 +150,7 @@ def genmixdb(
     from sonusai.mixture import AugmentationRule
     from sonusai.mixture import MixtureDatabase
     from sonusai.mixture import balance_targets
+    from sonusai.mixture import generate_mixtures
     from sonusai.mixture import get_all_snrs_from_config
     from sonusai.mixture import get_augmentation_rules
     from sonusai.mixture import get_augmented_targets
@@ -316,8 +316,10 @@ def genmixdb(
             f"{seconds_to_hms(seconds=noise_audio_duration)}"
         )
 
-    used_noise_files, used_noise_samples = populate_mixture_table(
-        location=location,
+    if logging:
+        logger.info("Generating mixtures")
+
+    used_noise_files, used_noise_samples, mixtures = generate_mixtures(
         noise_mix_mode=mixdb.noise_mix_mode,
         augmented_targets=augmented_targets,
         target_files=target_files,
@@ -330,17 +332,16 @@ def genmixdb(
         num_classes=mixdb.num_classes,
         feature_step_samples=mixdb.feature_step_samples,
         num_ir=mixdb.num_impulse_response_files,
-        test=test,
     )
 
-    num_mixtures = len(mixdb.mixtures)
+    num_mixtures = len(mixtures)
     update_mixid_width(location, num_mixtures, test)
 
     if logging:
         logger.info("")
         logger.info(f"Found {num_mixtures:,} mixtures to process")
 
-    total_duration = float(sum([mixture.samples for mixture in mixdb.mixtures])) / SAMPLE_RATE
+    total_duration = float(sum([mixture.samples for mixture in mixtures])) / SAMPLE_RATE
 
     if logging:
         log_duration_and_sizes(
@@ -362,23 +363,29 @@ def genmixdb(
 
     # Fill in the details
     if logging:
-        logger.info("Generating mixtures")
+        logger.info("Processing mixtures")
     progress = track(total=num_mixtures, disable=not show_progress)
-    par_track(
+    mixtures = par_track(
         partial(
             _process_mixture,
             location=location,
             save_mix=save_mix,
-            save_ft=save_ft,
-            save_segsnr=save_segsnr,
             test=test,
         ),
-        range(num_mixtures),
+        mixtures,
         progress=progress,
         no_par=no_par,
     )
     progress.close()
 
+    populate_mixture_table(
+        location=location,
+        mixtures=mixtures,
+        test=test,
+        logging=logging,
+        show_progress=show_progress,
+    )
+
     total_noise_files = len(noise_files)
 
     total_samples = mixdb.total_samples()
@@ -409,32 +416,23 @@ def genmixdb(
 
 
 def _process_mixture(
-    m_id: int,
+    mixture: Mixture,
     location: str,
     save_mix: bool,
-    save_ft: bool,
-    save_segsnr: bool,
     test: bool,
-) -> None:
+) -> Mixture:
     from functools import partial
 
-    from sonusai.mixture import MixtureDatabase
-    from sonusai.mixture import clear_cached_data
-    from sonusai.mixture import update_mixture_table
+    from sonusai.mixture import update_mixture
     from sonusai.mixture import write_cached_data
     from sonusai.mixture import write_mixture_metadata
 
-    with_data = save_mix or save_ft or save_segsnr
-
-    genmix_data = update_mixture_table(location, m_id, with_data, test)
-
-    mixdb = MixtureDatabase(location, test)
-    mixture = mixdb.mixture(m_id)
+    mixdb = MixtureDatabase(location, test=test)
+    mixture, genmix_data = update_mixture(mixdb, mixture, save_mix)
 
     write = partial(write_cached_data, location=location, name="mixture", index=mixture.name)
-    clear = partial(clear_cached_data, location=location, name="mixture", index=mixture.name)
 
-    if with_data:
+    if save_mix:
         write(
             items=[
                 ("targets", genmix_data.targets),
@@ -444,25 +442,9 @@ def _process_mixture(
             ]
         )
 
-        if save_ft:
-            clear(items=["feature", "truth_f"])
-            feature, truth_f = mixdb.mixture_ft(m_id)
-            write(
-                items=[
-                    ("feature", feature),
-                    ("truth_f", truth_f),
-                ]
-            )
-
-        if save_segsnr:
-            clear(items=["segsnr"])
-            segsnr = mixdb.mixture_segsnr(m_id)
-            write(items=[("segsnr", segsnr)])
-
-        if not save_mix:
-            clear(items=["targets", "target", "noise", "mixture"])
+        write_mixture_metadata(mixdb, mixture=mixture)
 
-        write_mixture_metadata(mixdb, m_id)
+    return mixture
 
 
 def main() -> None:
@@ -491,8 +473,6 @@ def main() -> None:
 
     verbose = args["--verbose"]
     save_mix = args["--mix"]
-    save_ft = args["--ft"]
-    save_segsnr = args["--segsnr"]
     dryrun = args["--dryrun"]
     save_json = args["--json"]
     no_par = args["--nopar"]
@@ -522,8 +502,6 @@ def main() -> None:
         genmixdb(
             location=location,
             save_mix=save_mix,
-            save_ft=save_ft,
-            save_segsnr=save_segsnr,
             show_progress=True,
             save_json=save_json,
             no_par=no_par,