sonusai 0.20.2__py3-none-any.whl → 1.0.1__py3-none-any.whl

sonusai/mixture/generation.py

@@ -1,17 +1,14 @@
 # ruff: noqa: S608
-from .datatypes import AudioT
-from .datatypes import Augmentation
-from .datatypes import AugmentationRule
-from .datatypes import AugmentedTarget
-from .datatypes import GenMixData
-from .datatypes import ImpulseResponseFile
-from .datatypes import Mixture
-from .datatypes import NoiseFile
-from .datatypes import SpectralMask
-from .datatypes import Target
-from .datatypes import TargetFile
-from .datatypes import UniversalSNRGenerator
 from .mixdb import MixtureDatabase
+from ..datatypes import AudioT
+from ..datatypes import Effects
+from ..datatypes import GenMixData
+from ..datatypes import ImpulseResponseFile
+from ..datatypes import Mixture
+from ..datatypes import Source
+from ..datatypes import SourceFile
+from ..datatypes import SourcesAudioT
+from ..datatypes import UniversalSNRGenerator
 
 
 def config_file(location: str) -> str:
@@ -34,47 +31,62 @@ def initialize_db(location: str, test: bool = False) -> None:
     con.execute("""
     CREATE TABLE truth_parameters(
     id INTEGER PRIMARY KEY NOT NULL,
+    category TEXT NOT NULL,
     name TEXT NOT NULL,
     parameters INTEGER)
     """)
 
     con.execute("""
-    CREATE TABLE target_file (
+    CREATE TABLE source_file (
     id INTEGER PRIMARY KEY NOT NULL,
+    category TEXT NOT NULL,
+    class_indices TEXT,
+    level_type TEXT NOT NULL,
     name TEXT NOT NULL,
     samples INTEGER NOT NULL,
-    class_indices TEXT NOT NULL,
-    level_type TEXT NOT NULL,
     speaker_id INTEGER,
     FOREIGN KEY(speaker_id) REFERENCES speaker (id))
     """)
 
     con.execute("""
-    CREATE TABLE speaker (
+    CREATE TABLE ir_file (
     id INTEGER PRIMARY KEY NOT NULL,
-    parent TEXT NOT NULL)
+    delay INTEGER NOT NULL,
+    name TEXT NOT NULL)
     """)
 
     con.execute("""
-    CREATE TABLE noise_file (
+    CREATE TABLE ir_tag (
     id INTEGER PRIMARY KEY NOT NULL,
-    name TEXT NOT NULL,
-    samples INTEGER NOT NULL)
+    tag TEXT NOT NULL UNIQUE)
+    """)
+
+    con.execute("""
+    CREATE TABLE ir_file_ir_tag (
+    file_id INTEGER NOT NULL,
+    tag_id INTEGER NOT NULL,
+    FOREIGN KEY(file_id) REFERENCES ir_file (id),
+    FOREIGN KEY(tag_id) REFERENCES ir_tag (id))
+    """)
+
+    con.execute("""
+    CREATE TABLE speaker (
+    id INTEGER PRIMARY KEY NOT NULL,
+    parent TEXT NOT NULL)
     """)
 
     con.execute("""
     CREATE TABLE top (
     id INTEGER PRIMARY KEY NOT NULL,
-    version INTEGER NOT NULL,
     asr_configs TEXT NOT NULL,
     class_balancing BOOLEAN NOT NULL,
     feature TEXT NOT NULL,
-    noise_mix_mode TEXT NOT NULL,
+    mixid_width INTEGER NOT NULL,
     num_classes INTEGER NOT NULL,
     seed INTEGER NOT NULL,
-    mixid_width INTEGER NOT NULL,
     speaker_metadata_tiers TEXT NOT NULL,
-    textgrid_metadata_tiers TEXT NOT NULL)
+    textgrid_metadata_tiers TEXT NOT NULL,
+    version INTEGER NOT NULL)
     """)
 
     con.execute("""
@@ -89,64 +101,54 @@ def initialize_db(location: str, test: bool = False) -> None:
     threshold FLOAT NOT NULL)
     """)
 
-    con.execute("""
-    CREATE TABLE impulse_response_file (
-    id INTEGER PRIMARY KEY NOT NULL,
-    file TEXT NOT NULL,
-    tags TEXT NOT NULL,
-    delay INTEGER NOT NULL)
-    """)
-
     con.execute("""
     CREATE TABLE spectral_mask (
     id INTEGER PRIMARY KEY NOT NULL,
     f_max_width INTEGER NOT NULL,
     f_num INTEGER NOT NULL,
+    t_max_percent INTEGER NOT NULL,
     t_max_width INTEGER NOT NULL,
-    t_num INTEGER NOT NULL,
-    t_max_percent INTEGER NOT NULL)
+    t_num INTEGER NOT NULL)
     """)
 
     con.execute("""
-    CREATE TABLE target_file_truth_config (
-    target_file_id INTEGER,
-    truth_config_id INTEGER,
-    FOREIGN KEY(target_file_id) REFERENCES target_file (id),
+    CREATE TABLE source_file_truth_config (
+    source_file_id INTEGER NOT NULL,
+    truth_config_id INTEGER NOT NULL,
+    FOREIGN KEY(source_file_id) REFERENCES source_file (id),
     FOREIGN KEY(truth_config_id) REFERENCES truth_config (id))
     """)
 
     con.execute("""
-    CREATE TABLE target (
+    CREATE TABLE source (
     id INTEGER PRIMARY KEY NOT NULL,
+    effects TEXT NOT NULL,
     file_id INTEGER NOT NULL,
-    augmentation TEXT NOT NULL,
-    FOREIGN KEY(file_id) REFERENCES target_file (id))
+    pre_tempo FLOAT NOT NULL,
+    repeat BOOLEAN NOT NULL,
+    snr FLOAT NOT NULL,
+    snr_gain FLOAT NOT NULL,
+    snr_random BOOLEAN NOT NULL,
+    start INTEGER NOT NULL,
+    FOREIGN KEY(file_id) REFERENCES source_file (id))
     """)
 
     con.execute("""
     CREATE TABLE mixture (
     id INTEGER PRIMARY KEY NOT NULL,
-    name VARCHAR NOT NULL,
-    noise_file_id INTEGER NOT NULL,
-    noise_augmentation TEXT NOT NULL,
-    noise_offset INTEGER NOT NULL,
-    noise_snr_gain FLOAT,
-    random_snr BOOLEAN NOT NULL,
-    snr FLOAT NOT NULL,
+    name TEXT NOT NULL,
     samples INTEGER NOT NULL,
     spectral_mask_id INTEGER NOT NULL,
     spectral_mask_seed INTEGER NOT NULL,
-    target_snr_gain FLOAT,
-    FOREIGN KEY(noise_file_id) REFERENCES noise_file (id),
     FOREIGN KEY(spectral_mask_id) REFERENCES spectral_mask (id))
     """)
 
     con.execute("""
-    CREATE TABLE mixture_target (
-    mixture_id INTEGER,
-    target_id INTEGER,
+    CREATE TABLE mixture_source (
+    mixture_id INTEGER NOT NULL,
+    source_id INTEGER NOT NULL,
     FOREIGN KEY(mixture_id) REFERENCES mixture (id),
-    FOREIGN KEY(target_id) REFERENCES target (id))
+    FOREIGN KEY(source_id) REFERENCES source (id))
     """)
 
     con.commit()
@@ -163,22 +165,21 @@ def populate_top_table(location: str, config: dict, test: bool = False) -> None:
     con = db_connection(location=location, readonly=False, test=test)
     con.execute(
         """
-    INSERT INTO top (id, version, asr_configs, class_balancing, feature, noise_mix_mode, num_classes,
-    seed, mixid_width, speaker_metadata_tiers, textgrid_metadata_tiers)
-    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    INSERT INTO top (id, asr_configs, class_balancing, feature, mixid_width, num_classes,
+    seed, speaker_metadata_tiers, textgrid_metadata_tiers, version)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
     """,
         (
             1,
-            MIXDB_VERSION,
             json.dumps(config["asr_configs"]),
             config["class_balancing"],
             config["feature"],
-            config["noise_mix_mode"],
+            0,
             config["num_classes"],
             config["seed"],
-            0,
             "",
             "",
+            MIXDB_VERSION,
         ),
     )
     con.commit()
@@ -231,15 +232,15 @@ def populate_spectral_mask_table(location: str, config: dict, test: bool = False
     con = db_connection(location=location, readonly=False, test=test)
     con.executemany(
         """
-    INSERT INTO spectral_mask (f_max_width, f_num, t_max_width, t_num, t_max_percent) VALUES (?, ?, ?, ?, ?)
+    INSERT INTO spectral_mask (f_max_width, f_num, t_max_percent, t_max_width, t_num) VALUES (?, ?, ?, ?, ?)
     """,
         [
             (
                 item.f_max_width,
                 item.f_num,
+                item.t_max_percent,
                 item.t_max_width,
                 item.t_num,
-                item.t_max_percent,
             )
             for item in get_spectral_masks(config)
         ],
@@ -256,10 +257,11 @@ def populate_truth_parameters_table(location: str, config: dict, test: bool = Fa
     con = db_connection(location=location, readonly=False, test=test)
     con.executemany(
         """
-    INSERT INTO truth_parameters (name, parameters) VALUES (?, ?)
+    INSERT INTO truth_parameters (category, name, parameters) VALUES (?, ?, ?)
     """,
         [
             (
+                item.category,
                 item.name,
                 item.parameters,
             )
@@ -270,40 +272,41 @@ def populate_truth_parameters_table(location: str, config: dict, test: bool = Fa
     con.close()
 
 
-def populate_target_file_table(location: str, target_files: list[TargetFile], test: bool = False) -> None:
-    """Populate target file table"""
+def populate_source_file_table(location: str, files: list[SourceFile], test: bool = False) -> None:
+    """Populate source file table"""
     import json
     from pathlib import Path
 
     from .mixdb import db_connection
 
-    _populate_truth_config_table(location, target_files, test)
-    _populate_speaker_table(location, target_files, test)
+    _populate_truth_config_table(location, files, test)
+    _populate_speaker_table(location, files, test)
 
     con = db_connection(location=location, readonly=False, test=test)
 
     cur = con.cursor()
     textgrid_metadata_tiers: set[str] = set()
-    for target_file in target_files:
-        # Get TextGrid tiers for target file and add to collection
-        tiers = _get_textgrid_tiers_from_target_file(target_file.name)
+    for file in files:
+        # Get TextGrid tiers for source file and add to collection
+        tiers = _get_textgrid_tiers_from_source_file(file.name)
         for tier in tiers:
             textgrid_metadata_tiers.add(tier)
 
-        # Get truth settings for target file
+        # Get truth settings for file
         truth_config_ids: list[int] = []
-        for name, config in target_file.truth_configs.items():
-            ts = json.dumps({"name": name} | config.to_dict())
-            cur.execute(
-                "SELECT truth_config.id FROM truth_config WHERE ? = truth_config.config",
-                (ts,),
-            )
-            truth_config_ids.append(cur.fetchone()[0])
-
-        # Get speaker_id for target file
+        if file.truth_configs:
+            for name, config in file.truth_configs.items():
+                ts = json.dumps({"name": name} | config.to_dict())
+                cur.execute(
+                    "SELECT truth_config.id FROM truth_config WHERE ? = truth_config.config",
+                    (ts,),
+                )
+                truth_config_ids.append(cur.fetchone()[0])
+
+        # Get speaker_id for source file
         cur.execute(
             "SELECT speaker.id FROM speaker WHERE ? = speaker.parent",
-            (Path(target_file.name).parent.as_posix(),),
+            (Path(file.name).parent.as_posix(),),
         )
         result = cur.fetchone()
         speaker_id = None
@@ -312,20 +315,24 @@ def populate_target_file_table(location: str, target_files: list[TargetFile], te
 
         # Add entry
         cur.execute(
-            "INSERT INTO target_file (name, samples, class_indices, level_type, speaker_id) VALUES (?, ?, ?, ?, ?)",
+            """
+            INSERT INTO source_file (category, class_indices, level_type, name, samples, speaker_id)
+            VALUES (?, ?, ?, ?, ?, ?)
+            """,
             (
-                target_file.name,
-                target_file.samples,
-                json.dumps(target_file.class_indices),
-                target_file.level_type,
+                file.category,
+                json.dumps(file.class_indices),
+                file.level_type,
+                file.name,
+                file.samples,
                 speaker_id,
             ),
         )
-        target_file_id = cur.lastrowid
+        source_file_id = cur.lastrowid
         for truth_config_id in truth_config_ids:
             cur.execute(
-                "INSERT INTO target_file_truth_config (target_file_id, truth_config_id) VALUES (?, ?)",
-                (target_file_id, truth_config_id),
+                "INSERT INTO source_file_truth_config (source_file_id, truth_config_id) VALUES (?, ?)",
+                (source_file_id, truth_config_id),
             )
 
     # Update textgrid_metadata_tiers in the top table
@@ -338,47 +345,47 @@ def populate_target_file_table(location: str, target_files: list[TargetFile], te
     con.close()
 
 
-def populate_noise_file_table(location: str, noise_files: list[NoiseFile], test: bool = False) -> None:
-    """Populate noise file table"""
+def populate_impulse_response_file_table(location: str, files: list[ImpulseResponseFile], test: bool = False) -> None:
+    """Populate impulse response file table"""
     from .mixdb import db_connection
 
-    con = db_connection(location=location, readonly=False, test=test)
-    con.executemany(
-        "INSERT INTO noise_file (name, samples) VALUES (?, ?)",
-        [(noise_file.name, noise_file.samples) for noise_file in noise_files],
-    )
-    con.commit()
-    con.close()
-
+    _populate_impulse_response_tag_table(location, files, test)
 
-def populate_impulse_response_file_table(
-    location: str, impulse_response_files: list[ImpulseResponseFile], test: bool = False
-) -> None:
-    """Populate impulse response file table"""
-    import json
+    con = db_connection(location=location, readonly=False, test=test)
 
-    from .mixdb import db_connection
+    cur = con.cursor()
+    for file in files:
+        # Get tags for file
+        tag_ids: list[int] = []
+        for tag in file.tags:
+            cur.execute(
+                "SELECT ir_tag.id FROM ir_tag WHERE ? = ir_tag.tag",
+                (tag,),
+            )
+            tag_ids.append(cur.fetchone()[0])
 
-    con = db_connection(location=location, readonly=False, test=test)
-    con.executemany(
-        "INSERT INTO impulse_response_file (file, tags, delay) VALUES (?, ?, ?)",
-        [
+        cur.execute(
+            "INSERT INTO ir_file (delay, name) VALUES (?, ?)",
             (
-                impulse_response_file.file,
-                json.dumps(impulse_response_file.tags),
-                impulse_response_file.delay,
+                file.delay,
+                file.name,
+            ),
+        )
+
+        file_id = cur.lastrowid
+        for tag_id in tag_ids:
+            cur.execute(
+                "INSERT INTO ir_file_ir_tag (file_id, tag_id) VALUES (?, ?)",
+                (file_id, tag_id),
             )
-            for impulse_response_file in impulse_response_files
-        ],
-    )
+
     con.commit()
     con.close()
 
 
 def update_mixid_width(location: str, num_mixtures: int, test: bool = False) -> None:
     """Update the mixid width"""
-    from sonusai.utils import max_text_width
-
+    from ..utils.max_text_width import max_text_width
     from .mixdb import db_connection
 
     con = db_connection(location=location, readonly=False, test=test)
@@ -391,42 +398,43 @@ def update_mixid_width(location: str, num_mixtures: int, test: bool = False) ->
 
 
 def generate_mixtures(
-    noise_mix_mode: str,
-    augmented_targets: list[AugmentedTarget],
-    target_files: list[TargetFile],
-    target_augmentations: list[AugmentationRule],
-    noise_files: list[NoiseFile],
-    noise_augmentations: list[AugmentationRule],
-    spectral_masks: list[SpectralMask],
-    all_snrs: list[UniversalSNRGenerator],
-    mixups: list[int],
-    num_classes: int,
-    feature_step_samples: int,
-    num_ir: int,
-) -> tuple[int, int, list[Mixture]]:
+    location: str,
+    config: dict,
+    effects: dict[str, list[Effects]],
+    test: bool = False,
+) -> list[Mixture]:
     """Generate mixtures"""
-    if noise_mix_mode == "exhaustive":
-        func = _exhaustive_noise_mix
-    elif noise_mix_mode == "non-exhaustive":
-        func = _non_exhaustive_noise_mix
-    elif noise_mix_mode == "non-combinatorial":
-        func = _non_combinatorial_noise_mix
-    else:
-        raise ValueError(f"invalid noise_mix_mode: {noise_mix_mode}")
-
-    return func(
-        augmented_targets=augmented_targets,
-        target_files=target_files,
-        target_augmentations=target_augmentations,
-        noise_files=noise_files,
-        noise_augmentations=noise_augmentations,
-        spectral_masks=spectral_masks,
-        all_snrs=all_snrs,
-        mixups=mixups,
-        num_classes=num_classes,
-        feature_step_samples=feature_step_samples,
-        num_ir=num_ir,
-    )
+    mixdb = MixtureDatabase(location, test)
+
+    effected_sources: dict[str, list[tuple[SourceFile, Effects]]] = {}
+    for category in mixdb.source_files:
+        effected_sources[category] = []
+        for file in mixdb.source_files[category]:
+            for effect in effects[category]:
+                effected_sources[category].append((file, effect))
+
+    mixtures: list[Mixture] = []
+    for noise_mix_rule in config["sources"]["noise"]["mix_rules"]:
+        match noise_mix_rule["mode"]:
+            case "exhaustive":
+                func = _exhaustive_noise_mix
+            case "non-exhaustive":
+                func = _non_exhaustive_noise_mix
+            case "non-combinatorial":
+                func = _non_combinatorial_noise_mix
+            case _:
+                raise ValueError(f"invalid noise mix_rule mode: {noise_mix_rule['mode']}")
+
+        mixtures.extend(
+            func(
+                location=location,
+                config=config,
+                effected_sources=effected_sources,
+                test=test,
+            )
+        )
+
+    return mixtures
 
 
 def populate_mixture_table(
@@ -437,26 +445,33 @@ def populate_mixture_table(
     show_progress: bool = False,
 ) -> None:
     """Populate mixture table"""
-    from sonusai import logger
-    from sonusai.utils import track
-
+    from .. import logger
+    from ..utils.parallel import track
     from .helpers import from_mixture
-    from .helpers import from_target
+    from .helpers import from_source
     from .mixdb import db_connection
 
     con = db_connection(location=location, readonly=False, test=test)
 
-    # Populate target table
+    # Populate source table
     if logging:
-        logger.info("Populating target table")
-    targets: list[tuple[int, str]] = []
+        logger.info("Populating source table")
+    sources: list[tuple[str, int, float, bool, float, float, bool, int]] = []
     for mixture in mixtures:
-        for target in mixture.targets:
-            entry = from_target(target)
-            if entry not in targets:
-                targets.append(entry)
-    for target in track(targets, disable=not show_progress):
-        con.execute("INSERT INTO target (file_id, augmentation) VALUES (?, ?)", target)
+        for source in mixture.all_sources.values():
+            entry = from_source(source)
+            if entry not in sources:
+                sources.append(entry)
+    for source in track(sources, disable=not show_progress):
+        con.execute(
+            """
+        INSERT INTO source (effects, file_id, pre_tempo, repeat, snr, snr_gain, snr_random, start)
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+        """,
+            source,
+        )
+
+    con.commit()
 
     # Populate mixture table
     if logging:
@@ -465,25 +480,31 @@ def populate_mixture_table(
         m_id = int(mixture.name)
         con.execute(
             """
-            INSERT INTO mixture (id, name, noise_file_id, noise_augmentation, noise_offset, noise_snr_gain, random_snr,
-            snr, samples, spectral_mask_id, spectral_mask_seed, target_snr_gain)
-            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+            INSERT INTO mixture (id, name, samples, spectral_mask_id, spectral_mask_seed)
+            VALUES (?, ?, ?, ?, ?)
             """,
             (m_id + 1, *from_mixture(mixture)),
         )
 
-        for target in mixture.targets:
-            target_id = con.execute(
+        for source in mixture.all_sources.values():
+            source_id = con.execute(
                 """
-                SELECT target.id
-                FROM target
-                WHERE ? = target.file_id AND ? = target.augmentation
+                SELECT source.id
+                FROM source
+                WHERE ? = source.effects
+                AND ? = source.file_id
+                AND ? = source.pre_tempo
+                AND ? = source.repeat
+                AND ? = source.snr
+                AND ? = source.snr_gain
+                AND ? = source.snr_random
+                AND ? = source.start
             """,
-                from_target(target),
+                from_source(source),
             ).fetchone()[0]
             con.execute(
-                "INSERT INTO mixture_target (mixture_id, target_id) VALUES (?, ?)",
-                (m_id + 1, target_id),
+                "INSERT INTO mixture_source (mixture_id, source_id) VALUES (?, ?)",
+                (m_id + 1, source_id),
             )
 
     con.commit()
@@ -491,525 +512,362 @@ def populate_mixture_table(
 
 
 def update_mixture(mixdb: MixtureDatabase, mixture: Mixture, with_data: bool = False) -> tuple[Mixture, GenMixData]:
-    """Update mixture record with name and gains"""
-    from .audio import get_next_noise
-    from .augmentation import apply_gain
-    from .helpers import get_target
-
-    mixture, targets_audio = _initialize_targets_audio(mixdb, mixture)
-
-    noise_audio = _augmented_noise_audio(mixdb, mixture)
-    noise_audio = get_next_noise(audio=noise_audio, offset=mixture.noise_offset, length=mixture.samples)
+    """Update mixture record with name, samples, and gains"""
+    import numpy as np
 
-    # Apply IR and sum targets audio before initializing the mixture SNR gains
-    target_audio = get_target(mixdb, mixture, targets_audio)
+    sources_audio: SourcesAudioT = {}
+    post_audio: SourcesAudioT = {}
+    for category in mixture.all_sources:
+        mixture, sources_audio[category], post_audio[category] = _update_source(mixdb, mixture, category)
 
-    mixture = _initialize_mixture_gains(
-        mixdb=mixdb, mixture=mixture, target_audio=target_audio, noise_audio=noise_audio
-    )
+    mixture = _initialize_mixture_gains(mixdb, mixture, post_audio)
 
     mixture.name = f"{int(mixture.name):0{mixdb.mixid_width}}"
 
     if not with_data:
         return mixture, GenMixData()
 
-    # Apply SNR gains
-    targets_audio = [apply_gain(audio=target_audio, gain=mixture.target_snr_gain) for target_audio in targets_audio]
-    noise_audio = apply_gain(audio=noise_audio, gain=mixture.noise_snr_gain)
+    # Apply gains
+    post_audio = {
+        category: post_audio[category] * mixture.all_sources[category].snr_gain for category in mixture.all_sources
+    }
 
-    # Apply IR and sum targets audio after applying the mixture SNR gains
-    target_audio = get_target(mixdb, mixture, targets_audio)
-    mixture_audio = target_audio + noise_audio
+    # Sum sources, noise, and mixture
+    source_audio = np.sum([post_audio[category] for category in mixture.sources], axis=0)
+    noise_audio = post_audio["noise"]
+    mixture_audio = source_audio + noise_audio
 
     return mixture, GenMixData(
-        mixture=mixture_audio,
-        targets=targets_audio,
-        target=target_audio,
+        sources=sources_audio,
+        source=source_audio,
         noise=noise_audio,
+        mixture=mixture_audio,
     )
 
 
-def _augmented_noise_audio(mixdb: MixtureDatabase, mixture: Mixture) -> AudioT:
-    from .audio import read_audio
-    from .augmentation import apply_augmentation
-
-    noise = mixdb.noise_file(mixture.noise.file_id)
-    noise_augmentation = mixture.noise.augmentation
+def _update_source(mixdb: MixtureDatabase, mixture: Mixture, category: str) -> tuple[Mixture, AudioT, AudioT]:
+    from .effects import apply_effects
+    from .effects import conform_audio_to_length
 
-    audio = read_audio(noise.name)
-    audio = apply_augmentation(mixdb, audio, noise_augmentation.pre)
+    source = mixture.all_sources[category]
+    org_audio = mixdb.read_source_audio(source.file_id)
 
-    return audio
+    org_samples = len(org_audio)
+    pre_audio = apply_effects(mixdb, org_audio, source.effects, pre=True, post=False)
 
+    pre_samples = len(pre_audio)
+    mixture.all_sources[category].pre_tempo = org_samples / pre_samples
 
-def _initialize_targets_audio(mixdb: MixtureDatabase, mixture: Mixture) -> tuple[Mixture, list[AudioT]]:
-    from .augmentation import apply_augmentation
-    from .augmentation import pad_audio_to_length
+    pre_audio = conform_audio_to_length(pre_audio, mixture.samples, source.repeat, source.start)
 
-    targets_audio = []
-    for target in mixture.targets:
-        target_audio = mixdb.read_target_audio(target.file_id)
-        targets_audio.append(
-            apply_augmentation(
-                mixdb=mixdb,
-                audio=target_audio,
-                augmentation=target.augmentation.pre,
-                frame_length=mixdb.feature_step_samples,
-            )
-        )
-
-    mixture.samples = max([len(item) for item in targets_audio])
+    post_audio = apply_effects(mixdb, pre_audio, source.effects, pre=False, post=True)
+    if len(pre_audio) != len(post_audio):
+        raise RuntimeError(f"post-truth effects changed length: {source.effects.post}")
 
-    for idx in range(len(targets_audio)):
-        targets_audio[idx] = pad_audio_to_length(audio=targets_audio[idx], length=mixture.samples)
+    return mixture, pre_audio, post_audio
 
-    return mixture, targets_audio
 
-
-def _initialize_mixture_gains(
-    mixdb: MixtureDatabase,
-    mixture: Mixture,
-    target_audio: AudioT,
-    noise_audio: AudioT,
-) -> Mixture:
+def _initialize_mixture_gains(mixdb: MixtureDatabase, mixture: Mixture, sources_audio: SourcesAudioT) -> Mixture:
     import numpy as np
 
-    from sonusai.utils import asl_p56
-    from sonusai.utils import db_to_linear
-
-    if mixture.is_noise_only:
-        # Special case for zeroing out target data
-        mixture.target_snr_gain = 0
-        mixture.noise_snr_gain = 1
-    elif mixture.is_target_only:
-        # Special case for zeroing out noise data
-        mixture.target_snr_gain = 1
-        mixture.noise_snr_gain = 0
-    else:
-        target_level_types = [
-            target_file.level_type for target_file in [mixdb.target_file(target.file_id) for target in mixture.targets]
-        ]
-        if not all(level_type == target_level_types[0] for level_type in target_level_types):
-            raise ValueError("Not all target_level_types in mixup are the same")
-
-        level_type = target_level_types[0]
+    from ..utils.asl_p56 import asl_p56
+    from ..utils.db import db_to_linear
+
+    sources_energy: dict[str, float] = {}
+    for category in mixture.all_sources:
+        level_type = mixdb.source_file(mixture.all_sources[category].file_id).level_type
         match level_type:
             case "default":
-                target_energy = np.mean(np.square(target_audio))
+                sources_energy[category] = float(np.mean(np.square(sources_audio[category])))
             case "speech":
-                target_energy = asl_p56(target_audio)
+                sources_energy[category] = asl_p56(sources_audio[category])
             case _:
                 raise ValueError(f"Unknown level_type: {level_type}")
 
-        noise_energy = np.mean(np.square(noise_audio))
-        if noise_energy == 0:
-            noise_gain = 1
-        else:
-            noise_gain = np.sqrt(target_energy / noise_energy) / db_to_linear(mixture.snr)
-
-        # Check for noise_gain > 1 to avoid clipping
-        if noise_gain > 1:
-            mixture.target_snr_gain = 1 / noise_gain
-            mixture.noise_snr_gain = 1
-        else:
-            mixture.target_snr_gain = 1
-            mixture.noise_snr_gain = noise_gain
+    # Initialize all gains to 1
+    for category in mixture.all_sources:
+        mixture.all_sources[category].snr_gain = 1
+
+    # Resolve gains
+    for category in mixture.all_sources:
+        if mixture.is_noise_only and category != "noise":
+            # Special case for zeroing out source data
+            mixture.all_sources[category].snr_gain = 0
+        elif mixture.is_source_only and category == "noise":
+            # Special case for zeroing out noise data
+            mixture.all_sources[category].snr_gain = 0
+        elif category != "primary":
+            if sources_energy["primary"] == 0:
+                # Avoid divide-by-zero
+                mixture.all_sources[category].snr_gain = 1
+            else:
+                mixture.all_sources[category].snr_gain = float(
+                    np.sqrt(sources_energy["primary"] / sources_energy[category])
+                ) / db_to_linear(mixture.all_sources[category].snr)
+
+    # Normalize gains
+    max_snr_gain = max([source.snr_gain for source in mixture.all_sources.values()])
+    for category in mixture.all_sources:
+        mixture.all_sources[category].snr_gain = mixture.all_sources[category].snr_gain / max_snr_gain
 
     # Check for clipping in mixture
-    gain_adjusted_target_audio = target_audio * mixture.target_snr_gain
-    gain_adjusted_noise_audio = noise_audio * mixture.noise_snr_gain
-    mixture_audio = gain_adjusted_target_audio + gain_adjusted_noise_audio
-    max_abs_audio = max(abs(mixture_audio))
+    mixture_audio = np.sum(
+        [sources_audio[category] * mixture.all_sources[category].snr_gain for category in mixture.all_sources], axis=0
+    )
+    max_abs_audio = float(np.max(np.abs(mixture_audio)))
     clip_level = db_to_linear(-0.25)
     if max_abs_audio > clip_level:
-        # Clipping occurred; lower gains to bring audio within +/-1
         gain_adjustment = clip_level / max_abs_audio
-        mixture.target_snr_gain *= gain_adjustment
-        mixture.noise_snr_gain *= gain_adjustment
+        for category in mixture.all_sources:
+            mixture.all_sources[category].snr_gain *= gain_adjustment
+
+    # To improve repeatability, round results
+    for category in mixture.all_sources:
+        mixture.all_sources[category].snr_gain = round(mixture.all_sources[category].snr_gain, ndigits=5)
 
-    mixture.target_snr_gain = round(mixture.target_snr_gain, ndigits=5)
-    mixture.noise_snr_gain = round(mixture.noise_snr_gain, ndigits=5)
     return mixture
 
 
 def _exhaustive_noise_mix(
-    augmented_targets: list[AugmentedTarget],
-    target_files: list[TargetFile],
-    target_augmentations: list[AugmentationRule],
-    noise_files: list[NoiseFile],
-    noise_augmentations: list[AugmentationRule],
-    spectral_masks: list[SpectralMask],
-    all_snrs: list[UniversalSNRGenerator],
-    mixups: list[int],
-    num_classes: int,
-    feature_step_samples: int,
-    num_ir: int,
-) -> tuple[int, int, list[Mixture]]:
-    """Use every noise/augmentation with every target/augmentation+interferences/augmentation"""
+    location: str,
+    config: dict,
+    effected_sources: dict[str, list[tuple[SourceFile, Effects]]],
+    test: bool = False,
+) -> list[Mixture]:
+    """Use every noise/effect with every source/effect+interferences/effect"""
     from random import randint
 
     import numpy as np
 
-    from .augmentation import augmentation_from_rule
-    from .augmentation import estimate_augmented_length_from_length
-    from .datatypes import Mixture
-    from .datatypes import Noise
-    from .datatypes import UniversalSNR
-    from .targets import get_augmented_target_ids_for_mixup
+    from ..datatypes import Mixture
+    from ..datatypes import UniversalSNR
+    from .effects import effects_from_rules
+    from .effects import estimate_effected_length
 
-    m_id = 0
-    used_noise_files = len(noise_files) * len(noise_augmentations)
-    used_noise_samples = 0
-
-    augmented_target_ids_for_mixups = [
-        get_augmented_target_ids_for_mixup(
-            augmented_targets=augmented_targets,
-            targets=target_files,
-            target_augmentations=target_augmentations,
-            mixup=mixup,
-            num_classes=num_classes,
-        )
-        for mixup in mixups
-    ]
+    mixdb = MixtureDatabase(location, test)
+    snrs = get_all_snrs_from_config(config)
 
+    m_id = 0
     mixtures: list[Mixture] = []
-    for noise_file_id in range(len(noise_files)):
-        for noise_augmentation_rule in noise_augmentations:
-            noise_augmentation = augmentation_from_rule(noise_augmentation_rule, num_ir)
-            noise_offset = 0
-            noise_length = estimate_augmented_length_from_length(
-                length=noise_files[noise_file_id].samples,
-                tempo=noise_augmentation.pre.tempo,
-            )
+    for noise_file, noise_rule in effected_sources["noise"]:
+        noise_start = 0
+        noise_effect = effects_from_rules(mixdb, noise_rule)
+        noise_length = estimate_effected_length(noise_file.samples, noise_effect)
 
-            for augmented_target_ids_for_mixup in augmented_target_ids_for_mixups:
-                for augmented_target_ids in augmented_target_ids_for_mixup:
-                    targets, target_length = _get_target_info(
-                        augmented_target_ids=augmented_target_ids,
-                        augmented_targets=augmented_targets,
-                        target_files=target_files,
-                        target_augmentations=target_augmentations,
-                        feature_step_samples=feature_step_samples,
-                        num_ir=num_ir,
-                    )
+        for primary_file, primary_rule in effected_sources["primary"]:
+            primary_effect = effects_from_rules(mixdb, primary_rule)
+            primary_length = estimate_effected_length(primary_file.samples, primary_effect, mixdb.feature_step_samples)
 
-                    for spectral_mask_id in range(len(spectral_masks)):
-                        for snr in all_snrs:
-                            mixtures.append(
-                                Mixture(
-                                    targets=targets,
-                                    name=str(m_id),
-                                    noise=Noise(file_id=noise_file_id + 1, augmentation=noise_augmentation),
-                                    noise_offset=noise_offset,
-                                    samples=target_length,
+            for spectral_mask_id in range(len(config["spectral_masks"])):
+                for snr in snrs["noise"]:
+                    mixtures.append(
+                        Mixture(
+                            name=str(m_id),
+                            all_sources={
+                                "primary": Source(
+                                    file_id=primary_file.id,
+                                    effects=primary_effect,
+                                ),
+                                "noise": Source(
+                                    file_id=noise_file.id,
+                                    effects=noise_effect,
+                                    start=noise_start,
+                                    repeat=True,
                                     snr=UniversalSNR(value=snr.value, is_random=snr.is_random),
-                                    spectral_mask_id=spectral_mask_id + 1,
-                                    spectral_mask_seed=randint(0, np.iinfo("i").max),  # noqa: S311
-                                )
-                            )
-                            m_id += 1
-
-                            noise_offset = int((noise_offset + target_length) % noise_length)
-                            used_noise_samples += target_length
+                                ),
+                            },
+                            samples=primary_length,
+                            spectral_mask_id=spectral_mask_id + 1,
+                            spectral_mask_seed=randint(0, np.iinfo("i").max),  # noqa: S311
+                        )
+                    )
+                    noise_start = int((noise_start + primary_length) % noise_length)
+                    m_id += 1
 
-    return used_noise_files, used_noise_samples, mixtures
+    return mixtures
 
 
 def _non_exhaustive_noise_mix(
-    augmented_targets: list[AugmentedTarget],
-    target_files: list[TargetFile],
-    target_augmentations: list[AugmentationRule],
-    noise_files: list[NoiseFile],
-    noise_augmentations: list[AugmentationRule],
-    spectral_masks: list[SpectralMask],
-    all_snrs: list[UniversalSNRGenerator],
-    mixups: list[int],
-    num_classes: int,
-    feature_step_samples: int,
-    num_ir: int,
-) -> tuple[int, int, list[Mixture]]:
-    """Cycle through every target/augmentation+interferences/augmentation without necessarily using all
-    noise/augmentation combinations (reduced data set).
+    location: str,
+    config: dict,
+    effected_sources: dict[str, list[tuple[SourceFile, Effects]]],
+    test: bool = False,
+) -> list[Mixture]:
+    """Cycle through every source/effect+interferences/effect without necessarily using all
+    noise/effect combinations (reduced data set).
     """
     from random import randint
 
     import numpy as np
 
-    from .datatypes import Mixture
-    from .datatypes import Noise
-    from .datatypes import UniversalSNR
-    from .targets import get_augmented_target_ids_for_mixup
+    from ..datatypes import Mixture
+    from ..datatypes import UniversalSNR
+    from .effects import effects_from_rules
+    from .effects import estimate_effected_length
 
-    m_id = 0
-    used_noise_files = set()
-    used_noise_samples = 0
-    noise_file_id = None
-    noise_augmentation_id = None
-    noise_offset = None
-
-    augmented_target_indices_for_mixups = [
-        get_augmented_target_ids_for_mixup(
-            augmented_targets=augmented_targets,
-            targets=target_files,
-            target_augmentations=target_augmentations,
-            mixup=mixup,
-            num_classes=num_classes,
-        )
-        for mixup in mixups
-    ]
+    mixdb = MixtureDatabase(location, test)
+    snrs = get_all_snrs_from_config(config)
 
-    mixtures: list[Mixture] = []
-    for mixup in augmented_target_indices_for_mixups:
-        for augmented_target_indices in mixup:
-            targets, target_length = _get_target_info(
-                augmented_target_ids=augmented_target_indices,
-                augmented_targets=augmented_targets,
-                target_files=target_files,
-                target_augmentations=target_augmentations,
-                feature_step_samples=feature_step_samples,
-                num_ir=num_ir,
-            )
-
-            for spectral_mask_id in range(len(spectral_masks)):
-                for snr in all_snrs:
-                    (
-                        noise_file_id,
-                        noise_augmentation_id,
-                        noise_augmentation,
-                        noise_offset,
-                    ) = _get_next_noise_offset(
-                        noise_file_id=noise_file_id,
-                        noise_augmentation_id=noise_augmentation_id,
-                        noise_offset=noise_offset,
-                        target_length=target_length,
-                        noise_files=noise_files,
-                        noise_augmentations=noise_augmentations,
-                        num_ir=num_ir,
-                    )
-                    used_noise_samples += target_length
+    next_noise = NextNoise(mixdb, effected_sources["noise"])
 
-                    used_noise_files.add(f"{noise_file_id}_{noise_augmentation_id}")
-
-                    mixtures.append(
-                        Mixture(
-                            targets=targets,
-                            name=str(m_id),
-                            noise=Noise(file_id=noise_file_id + 1, augmentation=noise_augmentation),
-                            noise_offset=noise_offset,
-                            samples=target_length,
-                            snr=UniversalSNR(value=snr.value, is_random=snr.is_random),
-                            spectral_mask_id=spectral_mask_id + 1,
-                            spectral_mask_seed=randint(0, np.iinfo("i").max),  # noqa: S311
-                        )
+    m_id = 0
+    mixtures: list[Mixture] = []
+    for primary_file, primary_rule in effected_sources["primary"]:
+        primary_effect = effects_from_rules(mixdb, primary_rule)
+        primary_length = estimate_effected_length(primary_file.samples, primary_effect, mixdb.feature_step_samples)
+
+        for spectral_mask_id in range(len(config["spectral_masks"])):
+            for snr in snrs["noise"]:
+                noise_file_id, noise_effect, noise_start = next_noise.generate(primary_file.samples)
+
+                mixtures.append(
+                    Mixture(
+                        name=str(m_id),
+                        all_sources={
+                            "primary": Source(
+                                file_id=primary_file.id,
+                                effects=primary_effect,
+                            ),
+                            "noise": Source(
+                                file_id=noise_file_id,
+                                effects=noise_effect,
+                                start=noise_start,
+                                repeat=True,
+                                snr=UniversalSNR(value=snr.value, is_random=snr.is_random),
+                            ),
+                        },
+                        samples=primary_length,
+                        spectral_mask_id=spectral_mask_id + 1,
+                        spectral_mask_seed=randint(0, np.iinfo("i").max),  # noqa: S311
                     )
-                    m_id += 1
+                )
+                m_id += 1
 
-    return len(used_noise_files), used_noise_samples, mixtures
+    return mixtures
 
 
 def _non_combinatorial_noise_mix(
-    augmented_targets: list[AugmentedTarget],
-    target_files: list[TargetFile],
-    target_augmentations: list[AugmentationRule],
-    noise_files: list[NoiseFile],
-    noise_augmentations: list[AugmentationRule],
-    spectral_masks: list[SpectralMask],
-    all_snrs: list[UniversalSNRGenerator],
-    mixups: list[int],
-    num_classes: int,
-    feature_step_samples: int,
-    num_ir: int,
-) -> tuple[int, int, list[Mixture]]:
-    """Combine a target/augmentation+interferences/augmentation with a single cut of a noise/augmentation
-    non-exhaustively (each target/augmentation+interferences/augmentation does not use each noise/augmentation).
-    Cut has random start and loop back to beginning if end of noise/augmentation is reached.
+    location: str,
+    config: dict,
+    effected_sources: dict[str, list[tuple[SourceFile, Effects]]],
+    test: bool = False,
+) -> list[Mixture]:
+    """Combine a source/effect+interferences/effect with a single cut of a noise/effect
+    non-exhaustively (each source/effect+interferences/effect does not use each noise/effect).
+    Cut has random start and loop back to beginning if end of noise/effect is reached.
     """
     from random import choice
     from random import randint
 
     import numpy as np
 
-    from .datatypes import Mixture
-    from .datatypes import Noise
-    from .datatypes import UniversalSNR
-    from .targets import get_augmented_target_ids_for_mixup
+    from ..datatypes import Mixture
+    from ..datatypes import UniversalSNR
+    from .effects import effects_from_rules
+    from .effects import estimate_effected_length
 
-    m_id = 0
-    used_noise_files = set()
-    used_noise_samples = 0
-    noise_file_id = None
-    noise_augmentation_id = None
-
-    augmented_target_indices_for_mixups = [
-        get_augmented_target_ids_for_mixup(
-            augmented_targets=augmented_targets,
-            targets=target_files,
-            target_augmentations=target_augmentations,
-            mixup=mixup,
-            num_classes=num_classes,
-        )
-        for mixup in mixups
-    ]
+    mixdb = MixtureDatabase(location, test)
+    snrs = get_all_snrs_from_config(config)
 
+    m_id = 0
+    noise_id = 0
     mixtures: list[Mixture] = []
-    for mixup in augmented_target_indices_for_mixups:
-        for augmented_target_indices in mixup:
-            targets, target_length = _get_target_info(
-                augmented_target_ids=augmented_target_indices,
-                augmented_targets=augmented_targets,
-                target_files=target_files,
-                target_augmentations=target_augmentations,
-                feature_step_samples=feature_step_samples,
-                num_ir=num_ir,
-            )
-
-            for spectral_mask_id in range(len(spectral_masks)):
-                for snr in all_snrs:
-                    (
-                        noise_file_id,
-                        noise_augmentation_id,
-                        noise_augmentation,
-                        noise_length,
-                    ) = _get_next_noise_indices(
-                        noise_file_id=noise_file_id,
-                        noise_augmentation_id=noise_augmentation_id,
-                        noise_files=noise_files,
-                        noise_augmentations=noise_augmentations,
-                        num_ir=num_ir,
+    for primary_file, primary_rule in effected_sources["primary"]:
+        primary_effect = effects_from_rules(mixdb, primary_rule)
+        primary_length = estimate_effected_length(primary_file.samples, primary_effect, mixdb.feature_step_samples)
+
+        for spectral_mask_id in range(len(config["spectral_masks"])):
+            for snr in snrs["noise"]:
+                noise_file, noise_rule = effected_sources["noise"][noise_id]
+                noise_effect = effects_from_rules(mixdb, noise_rule)
+                noise_length = estimate_effected_length(noise_file.samples, noise_effect)
+
+                mixtures.append(
+                    Mixture(
+                        name=str(m_id),
+                        all_sources={
+                            "primary": Source(
+                                file_id=primary_file.id,
+                                effects=primary_effect,
+                            ),
+                            "noise": Source(
+                                file_id=noise_file.id,
+                                effects=noise_effect,
+                                start=choice(range(noise_length)),  # noqa: S311
+                                repeat=True,
+                                snr=UniversalSNR(value=snr.value, is_random=snr.is_random),
+                            ),
+                        },
+                        samples=primary_length,
+                        spectral_mask_id=spectral_mask_id + 1,
+                        spectral_mask_seed=randint(0, np.iinfo("i").max),  # noqa: S311
                     )
-                    used_noise_samples += target_length
+                )
+                noise_id = (noise_id + 1) % len(effected_sources["noise"])
+                m_id += 1
 
-                    used_noise_files.add(f"{noise_file_id}_{noise_augmentation_id}")
+    return mixtures
 
-                    mixtures.append(
-                        Mixture(
-                            targets=targets,
-                            name=str(m_id),
-                            noise=Noise(file_id=noise_file_id + 1, augmentation=noise_augmentation),
-                            noise_offset=choice(range(noise_length)),  # noqa: S311
-                            samples=target_length,
-                            snr=UniversalSNR(value=snr.value, is_random=snr.is_random),
-                            spectral_mask_id=spectral_mask_id + 1,
-                            spectral_mask_seed=randint(0, np.iinfo("i").max),  # noqa: S311
-                        )
-                    )
-                    m_id += 1
 
-    return len(used_noise_files), used_noise_samples, mixtures
-
-
-def _get_next_noise_indices(
-    noise_file_id: int | None,
-    noise_augmentation_id: int | None,
-    noise_files: list[NoiseFile],
-    noise_augmentations: list[AugmentationRule],
-    num_ir: int,
-) -> tuple[int, int, Augmentation, int]:
-    from .augmentation import augmentation_from_rule
-    from .augmentation import estimate_augmented_length_from_length
-
-    if noise_file_id is None or noise_augmentation_id is None:
-        noise_file_id = 0
-        noise_augmentation_id = 0
-    else:
-        noise_augmentation_id += 1
-        if noise_augmentation_id == len(noise_augmentations):
-            noise_augmentation_id = 0
-            noise_file_id += 1
-            if noise_file_id == len(noise_files):
-                noise_file_id = 0
-
-    noise_augmentation = augmentation_from_rule(noise_augmentations[noise_augmentation_id], num_ir)
-    noise_length = estimate_augmented_length_from_length(
-        length=noise_files[noise_file_id].samples, tempo=noise_augmentation.pre.tempo
-    )
-    return noise_file_id, noise_augmentation_id, noise_augmentation, noise_length
-
-
-def _get_next_noise_offset(
-    noise_file_id: int | None,
-    noise_augmentation_id: int | None,
-    noise_offset: int | None,
-    target_length: int,
-    noise_files: list[NoiseFile],
-    noise_augmentations: list[AugmentationRule],
-    num_ir: int,
-) -> tuple[int, int, Augmentation, int]:
-    from .augmentation import augmentation_from_rule
-    from .augmentation import estimate_augmented_length_from_length
-
-    if noise_file_id is None or noise_augmentation_id is None or noise_offset is None:
-        noise_file_id = 0
-        noise_augmentation_id = 0
-        noise_offset = 0
-
-    noise_augmentation = augmentation_from_rule(noise_augmentations[noise_file_id], num_ir)
-    noise_length = estimate_augmented_length_from_length(
-        length=noise_files[noise_file_id].samples, tempo=noise_augmentation.pre.tempo
-    )
-    if noise_offset + target_length >= noise_length:
-        if noise_offset == 0:
-            raise ValueError("Length of target audio exceeds length of noise audio")
-
-        noise_offset = 0
-        noise_augmentation_id += 1
-        if noise_augmentation_id == len(noise_augmentations):
-            noise_augmentation_id = 0
-            noise_file_id += 1
-            if noise_file_id == len(noise_files):
-                noise_file_id = 0
-        noise_augmentation = augmentation_from_rule(noise_augmentations[noise_augmentation_id], num_ir)
-
-    return noise_file_id, noise_augmentation_id, noise_augmentation, noise_offset
-
-
-def _get_target_info(
-    augmented_target_ids: list[int],
-    augmented_targets: list[AugmentedTarget],
-    target_files: list[TargetFile],
-    target_augmentations: list[AugmentationRule],
-    feature_step_samples: int,
-    num_ir: int,
-) -> tuple[list[Target], int]:
-    from .augmentation import augmentation_from_rule
-    from .augmentation import estimate_augmented_length_from_length
-
-    mixups: list[Target] = []
-    target_length = 0
-    for idx in augmented_target_ids:
-        tfi = augmented_targets[idx].target_id
-        target_augmentation_rule = target_augmentations[augmented_targets[idx].target_augmentation_id]
-        target_augmentation = augmentation_from_rule(target_augmentation_rule, num_ir)
-
-        mixups.append(Target(file_id=tfi + 1, augmentation=target_augmentation))
-
-        target_length = max(
-            estimate_augmented_length_from_length(
-                length=target_files[tfi].samples,
-                tempo=target_augmentation.pre.tempo,
-                frame_length=feature_step_samples,
-            ),
-            target_length,
-        )
-    return mixups, target_length
+class NextNoise:
+    def __init__(self, mixdb: MixtureDatabase, effected_noises: list[tuple[SourceFile, Effects]]) -> None:
+        from .effects import effects_from_rules
+        from .effects import estimate_effected_length
 
+        self.mixdb = mixdb
+        self.effected_noises = effected_noises
 
-def get_all_snrs_from_config(config: dict) -> list[UniversalSNRGenerator]:
-    from .datatypes import UniversalSNRGenerator
+        self.noise_start = 0
+        self.noise_id = 0
+        self.noise_effect = effects_from_rules(self.mixdb, self.noise_rule)
+        self.noise_length = estimate_effected_length(self.noise_file.samples, self.noise_effect)
 
-    return [UniversalSNRGenerator(is_random=False, _raw_value=snr) for snr in config["snrs"]] + [
-        UniversalSNRGenerator(is_random=True, _raw_value=snr) for snr in config["random_snrs"]
-    ]
+    @property
+    def noise_file(self):
+        return self.effected_noises[self.noise_id][0]
+
+    @property
+    def noise_rule(self):
+        return self.effected_noises[self.noise_id][1]
 
+    def generate(self, length: int) -> tuple[int, Effects, int]:
+        from .effects import effects_from_rules
+        from .effects import estimate_effected_length
+
+        if self.noise_start + length > self.noise_length:
+            # Not enough samples in current noise
+            if self.noise_start == 0:
+                raise ValueError("Length of primary audio exceeds length of noise audio")
+
+            self.noise_start = 0
+            self.noise_id = (self.noise_id + 1) % len(self.effected_noises)
+            self.noise_effect = effects_from_rules(self.mixdb, self.noise_rule)
+            self.noise_length = estimate_effected_length(self.noise_file.samples, self.noise_effect)
+            noise_start = self.noise_start
+        else:
+            # Current noise has enough samples
+            noise_start = self.noise_start
+            self.noise_start += length
 
-def _get_textgrid_tiers_from_target_file(target_file: str) -> list[str]:
+        return self.noise_file.id, self.noise_effect, noise_start
+
+
+def get_all_snrs_from_config(config: dict) -> dict[str, list[UniversalSNRGenerator]]:
+    snrs: dict[str, list[UniversalSNRGenerator]] = {}
+    for category in config["sources"]:
+        if category != "primary":
+            snrs[category] = [UniversalSNRGenerator(snr) for snr in config["sources"][category]["snrs"]]
+    return snrs
+
+
+def _get_textgrid_tiers_from_source_file(file: str) -> list[str]:
     from pathlib import Path
 
     from praatio import textgrid
 
-    from sonusai.mixture import tokenized_expand
+    from ..utils.tokenized_shell_vars import tokenized_expand
 
-    textgrid_file = Path(tokenized_expand(target_file)[0]).with_suffix(".TextGrid")
+    textgrid_file = Path(tokenized_expand(file)[0]).with_suffix(".TextGrid")
     if not textgrid_file.exists():
         return []
 
@@ -1018,18 +876,18 @@ def _get_textgrid_tiers_from_target_file(target_file: str) -> list[str]:
     return sorted(tg.tierNames)
 
 
-def _populate_speaker_table(location: str, target_files: list[TargetFile], test: bool = False) -> None:
+def _populate_speaker_table(location: str, source_files: list[SourceFile], test: bool = False) -> None:
     """Populate speaker table"""
     import json
     from pathlib import Path
 
     import yaml
 
+    from ..utils.tokenized_shell_vars import tokenized_expand
     from .mixdb import db_connection
-    from .tokenized_shell_vars import tokenized_expand
 
     # Determine columns for speaker table
-    all_parents = {Path(target_file.name).parent for target_file in target_files}
+    all_parents = {Path(file.name).parent for file in source_files}
     speaker_parents = (parent for parent in all_parents if Path(tokenized_expand(parent / "speaker.yml")[0]).exists())
 
     speakers: dict[Path, dict[str, str]] = {}
@@ -1072,13 +930,13 @@ def _populate_speaker_table(location: str, target_files: list[TargetFile], test:
     )
 
     if "speaker_id" in tiers:
-        con.execute("CREATE INDEX speaker_speaker_id_idx ON speaker (speaker_id)")
+        con.execute("CREATE INDEX speaker_speaker_id_idx ON source_file (speaker_id)")
 
     con.commit()
     con.close()
 
 
-def _populate_truth_config_table(location: str, target_files: list[TargetFile], test: bool = False) -> None:
+def _populate_truth_config_table(location: str, source_files: list[SourceFile], test: bool = False) -> None:
     """Populate truth_config table"""
     import json
 
@@ -1088,8 +946,8 @@ def _populate_truth_config_table(location: str, target_files: list[TargetFile],
 
     # Populate truth_config table
     truth_configs: list[str] = []
-    for target_file in target_files:
-        for name, config in target_file.truth_configs.items():
+    for file in source_files:
+        for name, config in file.truth_configs.items():
             ts = json.dumps({"name": name} | config.to_dict())
             if ts not in truth_configs:
                 truth_configs.append(ts)
@@ -1100,3 +958,18 @@ def _populate_truth_config_table(location: str, target_files: list[TargetFile],
 
     con.commit()
     con.close()
+
+
+def _populate_impulse_response_tag_table(location: str, files: list[ImpulseResponseFile], test: bool = False) -> None:
+    """Populate ir_tag table"""
+    from .mixdb import db_connection
+
+    con = db_connection(location=location, readonly=False, test=test)
+
+    con.executemany(
+        "INSERT INTO ir_tag (tag) VALUES (?)",
+        [(tag,) for tag in {tag for file in files for tag in file.tags}],
+    )
+
+    con.commit()
+    con.close()