sonusai 1.0.16__cp311-abi3-macosx_10_12_x86_64.whl

sonusai/config/asr.py

@@ -0,0 +1,21 @@
+def validate_asr_configs(given: dict) -> None:
+    """Validate fields in 'asr_config' in the given config
+
+    :param given: The dictionary of the given config
+    """
+    from ..utils.asr import validate_asr
+    from .constants import REQUIRED_ASR_CONFIGS_FIELDS
+
+    if "asr_configs" not in given:
+        raise AttributeError("config is missing required 'asr_configs'")
+
+    asr_configs = given["asr_configs"]
+
+    for name, asr_config in asr_configs.items():
+        for key in REQUIRED_ASR_CONFIGS_FIELDS:
+            if key not in asr_config:
+                raise AttributeError(f"'{name}' in asr_configs is missing required '{key}'")
+
+        engine = asr_config["engine"]
+        config = {x: asr_config[x] for x in asr_config if x != "engine"}
+        validate_asr(engine, **config)

sonusai/config/config.py

@@ -0,0 +1,65 @@
+from functools import lru_cache
+
+
+def load_yaml(name: str) -> dict:
+    """Load YAML file
+
+    :param name: File name
+    :return: Dictionary of config data
+    """
+    import yaml
+
+    with open(file=name) as f:
+        config = yaml.safe_load(f)
+
+    return config
+
+
+@lru_cache
+def default_config() -> dict:
+    """Load default SonusAI config
+
+    :return: Dictionary of default config data
+    """
+    from .constants import DEFAULT_CONFIG
+
+    try:
+        return load_yaml(DEFAULT_CONFIG)
+    except Exception as e:
+        raise OSError(f"Error loading default config: {e}") from e
+
+
+def _update_config_from_file(filename: str, given_config: dict) -> dict:
+    """Update the given config with the config in the specified YAML file
+
+    :param filename: File name
+    :param given_config: Config dictionary to update
+    :return: Updated config dictionary
+    """
+    from copy import deepcopy
+
+    updated_config = deepcopy(given_config)
+
+    try:
+        file_config = load_yaml(filename)
+    except Exception as e:
+        raise OSError(f"Error loading config from {filename}: {e}") from e
+
+    # Use default config as base and overwrite with given config keys as found
+    if file_config:
+        for key in updated_config:
+            if key in file_config:
+                updated_config[key] = file_config[key]
+
+    return updated_config
+
+
+def load_config(name: str) -> dict:
+    """Load the SonusAI default config and update with the given location (performing SonusAI variable substitution)
+
+    :param name: Directory containing mixture database
+    :return: Dictionary of config data
+    """
+    from os.path import join
+
+    return _update_config_from_file(filename=join(name, "config.yml"), given_config=default_config())

sonusai/config/config.yml

@@ -0,0 +1,49 @@
+# Default configuration for sonusai
+
+# The values in this file are the defaults used if they are not specified in a
+# local config.
+
+feature: ""
+
+class_indices: 1
+
+num_classes: 1
+
+class_labels: [ ]
+
+seed: 0
+
+class_weights_threshold: 0.5
+
+asr_configs: { }
+
+class_balancing: false
+
+class_balancing_effect:
+  - norm -3.5
+  - pitch rand(-300, 300)
+  - tempo -s rand(0.8, 1.2)
+  - equalizer rand(50, 250) rand(0.2, 2.0) rand(-6, 6)
+  - equalizer rand(250, 1200) rand(0.2, 2.0) rand(-6, 6)
+  - equalizer rand(1200, 6000) rand(0.2, 2.0) rand(-6, 6)
+
+spectral_masks:
+  - f_max_width: 27
+    f_num: 0
+    t_max_width: 100
+    t_num: 0
+    t_max_percent: 100
+
+sources:
+  primary:
+    files: [ ]
+  noise:
+    files: [ ]
+
+level_type: default
+
+impulse_responses: [ ]
+
+summed_source_effects: [ ]
+
+mixture_effects: [ ]

sonusai/config/constants.py

@@ -0,0 +1,53 @@
+from importlib.resources import as_file
+from importlib.resources import files
+
+REQUIRED_CONFIGS: tuple[str, ...] = (
+    "asr_configs",
+    "class_balancing",
+    "class_balancing_effect",
+    "class_indices",
+    "class_labels",
+    "class_weights_threshold",
+    "feature",
+    "impulse_responses",
+    "level_type",
+    "mixture_effects",
+    "num_classes",
+    "seed",
+    "sources",
+    "spectral_masks",
+    "summed_source_effects",
+)
+OPTIONAL_CONFIGS: tuple[str, ...] = ()
+VALID_CONFIGS: tuple[str, ...] = REQUIRED_CONFIGS + OPTIONAL_CONFIGS
+
+REQUIRED_SOURCES_CATEGORIES: tuple[str, ...] = (
+    "primary",
+    "noise",
+)
+
+REQUIRED_SOURCE_CONFIG_FIELDS: tuple[str, ...] = (
+    "effects",
+    "files",
+)
+OPTIONAL_SOURCE_CONFIG_FIELDS: tuple[str, ...] = ("truth_configs",)
+REQUIRED_NON_PRIMARY_SOURCE_CONFIG_FIELDS: tuple[str, ...] = (
+    "mix_rules",
+    "snrs",
+)
+VALID_PRIMARY_SOURCE_CONFIG_FIELDS: tuple[str, ...] = REQUIRED_SOURCE_CONFIG_FIELDS + OPTIONAL_SOURCE_CONFIG_FIELDS
+VALID_NON_PRIMARY_SOURCE_CONFIG_FIELDS: tuple[str, ...] = (
+    VALID_PRIMARY_SOURCE_CONFIG_FIELDS + REQUIRED_NON_PRIMARY_SOURCE_CONFIG_FIELDS
+)
+
+REQUIRED_TRUTH_CONFIGS: tuple[str, ...] = (
+    "function",
+    "stride_reduction",
+)
+
+REQUIRED_ASR_CONFIGS_FIELDS: tuple[str, ...] = ("engine",)
+
+REQUIRED_TRUTH_CONFIG_FIELDS = ["function", "stride_reduction"]
+
+with as_file(files("sonusai.config").joinpath("config.yml")) as path:
+    DEFAULT_CONFIG = str(path)

sonusai/config/ir.py

@@ -0,0 +1,124 @@
+from sonusai.datatypes import ImpulseResponseFile
+
+
+def get_ir_files(config: dict, show_progress: bool = False) -> list[ImpulseResponseFile]:
+    """Get the list of impulse response files from a config
+
+    :param config: Config dictionary
+    :param show_progress: Show progress bar
+    :return: List of impulse response files
+    """
+    from itertools import chain
+
+    from ..utils.parallel import par_track
+    from ..utils.parallel import track
+
+    ir_files = list(
+        chain.from_iterable(
+            [
+                append_ir_files(
+                    entry=ImpulseResponseFile(
+                        name=entry["name"],
+                        tags=entry.get("tags", []),
+                        delay=entry.get("delay", "auto"),
+                    )
+                )
+                for entry in config["impulse_responses"]
+            ]
+        )
+    )
+
+    if len(ir_files) == 0:
+        return []
+
+    progress = track(total=len(ir_files), disable=not show_progress)
+    ir_files = par_track(_get_ir_delay, ir_files, progress=progress)
+    progress.close()
+
+    return ir_files
+
+
+def append_ir_files(entry: ImpulseResponseFile, tokens: dict | None = None) -> list[ImpulseResponseFile]:
+    """Process impulse response files list and append as needed
+
+    :param entry: Impulse response file entry to append to the list
+    :param tokens: Tokens used for variable expansion
+    :return: List of impulse response files
+    """
+    from glob import glob
+    from os import listdir
+    from os.path import dirname
+    from os.path import isabs
+    from os.path import isdir
+    from os.path import join
+    from os.path import splitext
+
+    from ..mixture.audio import validate_input_file
+    from ..utils.tokenized_shell_vars import tokenized_expand
+    from ..utils.tokenized_shell_vars import tokenized_replace
+    from .config import load_yaml
+
+    if tokens is None:
+        tokens = {}
+
+    in_name, new_tokens = tokenized_expand(entry.name)
+    tokens.update(new_tokens)
+    names = sorted(glob(in_name))
+    if not names:
+        raise OSError(f"Could not find {in_name}. Make sure path exists")
+
+    ir_files: list[ImpulseResponseFile] = []
+    for name in names:
+        ext = splitext(name)[1].lower()
+        dir_name = dirname(name)
+        if isdir(name):
+            for file in listdir(name):
+                if not isabs(file):
+                    file = join(dir_name, file)
+                child = ImpulseResponseFile(file, entry.tags, entry.delay)
+                ir_files.extend(append_ir_files(entry=child, tokens=tokens))
+        else:
+            try:
+                if ext == ".txt":
+                    with open(file=name) as txt_file:
+                        for line in txt_file:
+                            # strip comments
+                            file = line.partition("#")[0]
+                            file = file.rstrip()
+                            if file:
+                                file, new_tokens = tokenized_expand(file)
+                                tokens.update(new_tokens)
+                                if not isabs(file):
+                                    file = join(dir_name, file)
+                                child = ImpulseResponseFile(file, entry.tags, entry.delay)
+                                ir_files.extend(append_ir_files(entry=child, tokens=tokens))
+                elif ext == ".yml":
+                    try:
+                        yml_config = load_yaml(name)
+
+                        if "impulse_responses" in yml_config:
+                            for record in yml_config["impulse_responses"]:
+                                ir_files.extend(append_ir_files(entry=record, tokens=tokens))
+                    except Exception as e:
+                        raise OSError(f"Error processing {name}: {e}") from e
+                else:
+                    validate_input_file(name)
+                    ir_files.append(ImpulseResponseFile(tokenized_replace(name, tokens), entry.tags, entry.delay))
+            except Exception as e:
+                raise OSError(f"Error processing {name}: {e}") from e
+
+    return ir_files
+
+
+def _get_ir_delay(entry: ImpulseResponseFile) -> ImpulseResponseFile:
+    from .ir_delay import get_ir_delay
+
+    if entry.delay == "auto":
+        entry.delay = get_ir_delay(entry.name)
+    else:
+        try:
+            entry.delay = int(entry.delay)
+        except ValueError as e:
+            raise ValueError(f"Invalid impulse response delay: {entry.delay}") from e
+
+    return entry

sonusai/config/ir_delay.py

@@ -0,0 +1,62 @@
+import numpy as np
+
+
+def get_ir_delay(file: str) -> int:
+    from ..mixture.audio import raw_read_audio
+    from ..utils.rand import seed_context
+
+    ir, sample_rate = raw_read_audio(file)
+
+    with seed_context(42):
+        wgn_ref = np.random.normal(loc=0, scale=0.2, size=int(np.ceil(0.05 * sample_rate))).astype(np.float32)
+
+    wgn_conv = np.convolve(ir, wgn_ref)
+
+    return int(np.round(tdoa(wgn_conv, wgn_ref, interp=16, phat=True)))
+
+
+def tdoa(signal: np.ndarray, reference: np.ndarray, interp: int = 1, phat: bool = False, fs: int | float = 1) -> float:
+    """Estimates the shift of array signal with respect to reference using generalized cross-correlation.
+
+    :param signal: The array whose tdoa is measured
+    :param reference: The reference array
+    :param interp: Interpolation factor for the output array
+    :param phat: Apply the PHAT weighting
+    :param fs: The sampling frequency of the input arrays
+    :return: The estimated delay between the two arrays
+    """
+    n_reference = reference.shape[0]
+
+    r_12 = correlate(signal, reference, interp=interp, phat=phat)
+
+    delay = (np.argmax(np.abs(r_12)) / interp - (n_reference - 1)) / fs
+
+    return float(delay)
+
+
+def correlate(x1: np.ndarray, x2: np.ndarray, interp: int = 1, phat: bool = False) -> np.ndarray:
+    """Compute the cross-correlation between x1 and x2
+
+    :param x1: Input array 1
+    :param x2: Input array 2
+    :param interp: Interpolation factor for the output array
+    :param phat: Apply the PHAT weighting
+    :return: The cross-correlation between the two arrays
+    """
+    n_x1 = x1.shape[0]
+    n_x2 = x2.shape[0]
+
+    n = n_x1 + n_x2 - 1
+
+    fft1 = np.fft.rfft(x1, n=n)
+    fft2 = np.fft.rfft(x2, n=n)
+
+    if phat:
+        eps1 = np.mean(np.abs(fft1)) * 1e-10
+        fft1 /= np.abs(fft1) + eps1
+        eps2 = np.mean(np.abs(fft2)) * 1e-10
+        fft2 /= np.abs(fft2) + eps2
+
+    out = np.fft.irfft(fft1 * np.conj(fft2), n=int(n * interp))
+
+    return np.concatenate([out[-interp * (n_x2 - 1) :], out[: (interp * n_x1)]])

sonusai/config/source.py

@@ -0,0 +1,275 @@
+from sonusai.datatypes import SourceFile
+
+
+def update_sources(given: dict) -> dict:
+    """Validate and update fields in given 'sources'
+
+    :param given: The dictionary of the given config
+    """
+    from .constants import REQUIRED_NON_PRIMARY_SOURCE_CONFIG_FIELDS
+    from .constants import REQUIRED_SOURCE_CONFIG_FIELDS
+    from .constants import REQUIRED_SOURCES_CATEGORIES
+    from .constants import VALID_NON_PRIMARY_SOURCE_CONFIG_FIELDS
+    from .constants import VALID_PRIMARY_SOURCE_CONFIG_FIELDS
+
+    sources = given["sources"]
+
+    for category in REQUIRED_SOURCES_CATEGORIES:
+        if category not in sources:
+            raise AttributeError(f"config sources is missing required '{category}'")
+
+    for category, source in sources.items():
+        for key in REQUIRED_SOURCE_CONFIG_FIELDS:
+            if key not in source:
+                raise AttributeError(f"config source '{category}' is missing required '{key}'")
+
+        if category == "primary":
+            for key in source:
+                if key not in VALID_PRIMARY_SOURCE_CONFIG_FIELDS:
+                    nice_list = "\n".join([f"  {item}" for item in VALID_PRIMARY_SOURCE_CONFIG_FIELDS])
+                    raise AttributeError(
+                        f"Invalid source '{category}' config parameter: '{key}'.\nValid sources config parameters are:\n{nice_list}"
+                    )
+        else:
+            for key in REQUIRED_NON_PRIMARY_SOURCE_CONFIG_FIELDS:
+                if key not in source:
+                    raise AttributeError(f"config source '{category}' is missing required '{key}'")
+
+            for key in source:
+                if key not in VALID_NON_PRIMARY_SOURCE_CONFIG_FIELDS:
+                    nice_list = "\n".join([f"  {item}" for item in VALID_NON_PRIMARY_SOURCE_CONFIG_FIELDS])
+                    raise AttributeError(
+                        f"Invalid source '{category}' config parameter: '{key}'.\nValid source config parameters are:\n{nice_list}"
+                    )
+
+        files = source["files"]
+
+        if isinstance(files, str) and files in sources and files != category:
+            continue
+
+        if isinstance(files, list):
+            continue
+
+        raise TypeError(
+            f"'file' parameter of config source '{category}' is not a list or a reference to another source"
+        )
+
+    count = 0
+    while any(isinstance(source["files"], str) for source in sources.values()) and count < 100:
+        count += 1
+        for category, source in sources.items():
+            files = source["files"]
+            if isinstance(files, str):
+                given["sources"][category]["files"] = sources[files]["files"]
+
+    if count == 100:
+        raise RuntimeError("Check config sources for circular references")
+
+    return given
+
+
+def get_source_files(config: dict, show_progress: bool = False) -> list[SourceFile]:
+    """Get the list of source files from a config
+
+    :param config: Config dictionary
+    :param show_progress: Show progress bar
+    :return: List of source files
+    """
+    from itertools import chain
+
+    from ..utils.parallel import par_track
+    from ..utils.parallel import track
+
+    sources = config["sources"]
+    if not isinstance(sources, dict) and not all(isinstance(source, dict) for source in sources):
+        raise TypeError("'sources' must be a dictionary of dictionaries")
+
+    if "primary" not in sources:
+        raise AttributeError("'primary' is missing in 'sources'")
+
+    class_indices = config["class_indices"]
+    if not isinstance(class_indices, list):
+        class_indices = [class_indices]
+
+    level_type = config["level_type"]
+
+    source_files: list[SourceFile] = []
+    for category in sources:
+        source_files.extend(
+            chain.from_iterable(
+                [
+                    append_source_files(
+                        category=category,
+                        entry=entry,
+                        class_indices=class_indices,
+                        truth_configs=sources[category].get("truth_configs", []),
+                        level_type=level_type,
+                    )
+                    for entry in sources[category]["files"]
+                ]
+            )
+        )
+
+    progress = track(total=len(source_files), disable=not show_progress)
+    source_files = par_track(_get_num_samples, source_files, progress=progress)
+    progress.close()
+
+    num_classes = config["num_classes"]
+    for source_file in source_files:
+        if any(class_index < 0 for class_index in source_file.class_indices):
+            raise ValueError("class indices must contain only positive elements")
+
+        if any(class_index > num_classes for class_index in source_file.class_indices):
+            raise ValueError(f"class index elements must not be greater than {num_classes}")
+
+    return source_files
+
+
+def append_source_files(
+    category: str,
+    entry: dict,
+    class_indices: list[int],
+    truth_configs: dict,
+    level_type: str,
+    tokens: dict | None = None,
+) -> list[SourceFile]:
+    """Process source files list and append as needed
+
+    :param category: Source file category name
+    :param entry: Source file entry to append to the list
+    :param class_indices: Class indices
+    :param truth_configs: Truth configs
+    :param level_type: Level type
+    :param tokens: Tokens used for variable expansion
+    :return: List of source files
+    """
+    from copy import deepcopy
+    from glob import glob
+    from os import listdir
+    from os.path import dirname
+    from os.path import isabs
+    from os.path import isdir
+    from os.path import join
+    from os.path import splitext
+
+    from ..datatypes import TruthConfig
+    from ..mixture.audio import validate_input_file
+    from ..utils.dataclass_from_dict import dataclass_from_dict
+    from ..utils.tokenized_shell_vars import tokenized_expand
+    from ..utils.tokenized_shell_vars import tokenized_replace
+    from .constants import REQUIRED_TRUTH_CONFIG_FIELDS
+
+    if tokens is None:
+        tokens = {}
+
+    truth_configs_merged = deepcopy(truth_configs)
+
+    if not isinstance(entry, dict):
+        raise TypeError("'entry' must be a dictionary")
+
+    in_name = entry.get("name")
+    if in_name is None:
+        raise KeyError("Source file list contained record without name")
+
+    class_indices = entry.get("class_indices", class_indices)
+    if not isinstance(class_indices, list):
+        class_indices = [class_indices]
+
+    truth_configs_override = entry.get("truth_configs", {})
+    for key in truth_configs_override:
+        if key not in truth_configs:
+            raise AttributeError(
+                f"Truth config '{key}' override specified for {entry['name']} is not defined at top level"
+            )
+        if key in truth_configs_override:
+            truth_configs_merged[key] |= truth_configs_override[key]
+
+    level_type = entry.get("level_type", level_type)
+
+    in_name, new_tokens = tokenized_expand(in_name)
+    tokens.update(new_tokens)
+    names = sorted(glob(in_name))
+    if not names:
+        raise OSError(f"Could not find {in_name}. Make sure path exists")
+
+    source_files: list[SourceFile] = []
+    for name in names:
+        ext = splitext(name)[1].lower()
+        dir_name = dirname(name)
+        if isdir(name):
+            for file in listdir(name):
+                child = file
+                if not isabs(child):
+                    child = join(dir_name, child)
+                source_files.extend(
+                    append_source_files(
+                        category=category,
+                        entry={"name": child},
+                        class_indices=class_indices,
+                        truth_configs=truth_configs_merged,
+                        level_type=level_type,
+                        tokens=tokens,
+                    )
+                )
+        else:
+            try:
+                if ext == ".txt":
+                    with open(file=name) as txt_file:
+                        for line in txt_file:
+                            # strip comments
+                            child = line.partition("#")[0]
+                            child = child.rstrip()
+                            if child:
+                                child, new_tokens = tokenized_expand(child)
+                                tokens.update(new_tokens)
+                                if not isabs(child):
+                                    child = join(dir_name, child)
+                                source_files.extend(
+                                    append_source_files(
+                                        category=category,
+                                        entry={"name": child},
+                                        class_indices=class_indices,
+                                        truth_configs=truth_configs_merged,
+                                        level_type=level_type,
+                                        tokens=tokens,
+                                    )
+                                )
+                else:
+                    validate_input_file(name)
+                    source_file = SourceFile(
+                        category=category,
+                        name=tokenized_replace(name, tokens),
+                        samples=0,
+                        class_indices=class_indices,
+                        level_type=level_type,
+                        truth_configs={},
+                    )
+                    if len(truth_configs_merged) > 0:
+                        for tc_key, tc_value in truth_configs_merged.items():
+                            config = deepcopy(tc_value)
+                            truth_config: dict = {}
+                            for key in REQUIRED_TRUTH_CONFIG_FIELDS:
+                                truth_config[key] = config[key]
+                                del config[key]
+                            truth_config["config"] = config
+                            source_file.truth_configs[tc_key] = dataclass_from_dict(TruthConfig, truth_config)
+                        for tc_key in source_file.truth_configs:
+                            if (
+                                "function" in truth_configs_merged[tc_key]
+                                and truth_configs_merged[tc_key]["function"] == "file"
+                            ):
+                                truth_configs_merged[tc_key]["file"] = splitext(source_file.name)[0] + ".h5"
+                    source_files.append(source_file)
+            except Exception as e:
+                raise OSError(f"Error processing {name}: {e}") from e
+
+    return source_files
+
+
+def _get_num_samples(entry: SourceFile) -> SourceFile:
+    from ..mixture.audio import get_num_samples
+
+    entry.samples = get_num_samples(entry.name)
+    return entry
+
+

sonusai/config/spectral_masks.py

@@ -0,0 +1,15 @@
+from sonusai.datatypes import SpectralMask
+
+
+def get_spectral_masks(config: dict) -> list[SpectralMask]:
+    """Get the list of spectral masks from a config
+
+    :param config: Config dictionary
+    :return: List of spectral masks
+    """
+    from ..utils.dataclass_from_dict import list_dataclass_from_dict
+
+    try:
+        return list_dataclass_from_dict(list[SpectralMask], config["spectral_masks"])
+    except Exception as e:
+        raise ValueError(f"Error in spectral_masks: {e}") from e