sonusai 1.0.8__py3-none-any.whl → 1.0.9__py3-none-any.whl

sonusai/doc/doc.py

@@ -21,12 +21,12 @@ to use.
     # fmt: on
 
 
-def doc_target_level_type() -> str:
-    default = f"\nDefault value: {get_default_config()['target_level_type']}"
+def doc_level_type() -> str:
+    default = f"\nDefault value: {get_default_config()['level_type']}"
     # fmt: off
     return """
-'target_level_type' is a mixture database configuration parameter that sets the
-algorithm to use to determine target energy level for SNR calculations.
+'level_type' is a mixture database configuration parameter that sets the
+algorithm to use to determine energy level for SNR calculations.
 Supported values are:
 
  default    mean of squares
@@ -35,31 +35,31 @@ Supported values are:
     # fmt: on
 
 
-def doc_targets() -> str:
-    default = f"\nDefault value: {get_default_config()['targets']}"
+def doc_sources() -> str:
+    default = f"\nDefault value: {get_default_config()['sources']}"
     # fmt: off
     return """
-'targets' is a mixture database configuration parameter that sets the list of
-targets to use.
+'sources' is a mixture database configuration parameter that sets the list of
+sources to use.
 
-Required field:
-
- 'name'
-                File name. May be one of the following:
+Two sources are required: 'primary' and 'noise'. Additional sources may be
+specified with arbitrary names.
 
-  audio         Supported formats are .wav, .mp3, .m4a, .aif, .flac, and .ogg
-  glob          Matches file glob patterns
-  .yml          The given YAML file is parsed into the list
-  .txt          Each line in the given text file indicates an item which
-                may be anything in this list (audio, glob, .yml, or .txt)
+Each source has the following fields:
 
-Optional fields:
+ 'files'            Required list of files to use. Sub-fields:
+    'name'              File name. May be one of the following:
+            audio           Supported formats are .wav, .mp3, .m4a, .aif, .flac, and .ogg
+            glob            Matches file glob patterns
+            .yml            The given YAML file is parsed into the list
+            .txt            Each line in the given text file indicates an item which
+                            may be anything in this list (audio, glob, .yml, or .txt)
+    'class_indices'     Optional list of class indices
 
- 'truth_configs'
-                Local overrides for truth configs. Contains the following:
-  'name'        Name of truth config
-    '<param1>'  Target-specific override for truth configuration parameter
-    '<param2>'  Target-specific override for truth configuration parameter
+ 'truth_configs'    Required list of truth config(s) to use for this source. Sub-fields:
+    '<name>'            Name of truth config. Sub-fields:
+        'function'          Truth function
+        'stride_reduction'  Stride reduction method to use. May be one of: none, max
 
   'target_level_type'
                 Target-specific override for target_level_type.
@@ -72,7 +72,7 @@ targets:
       sed:
         thresholds: [-38, -41, -48]
         index: 2
-        class_balancing_augmentation: { }
+        class_balancing_effect: { }
   - name: target.mp3
     truth_configs:
       sed:
@@ -176,27 +176,27 @@ The 'truth_configs' parameter specifies the following:
                 Name of stride reduction method to use
     '<param1>'  Function-specific configuration parameter
     '<paramN>'  Function-specific configuration parameter
-    'class_balancing_augmentation'
-                Class balancing augmentation.
+    'class_balancing_effect'
+                Class balancing effect.
                 This truth configuration will use this rule for class balancing operations.
                 If this rule is empty or unspecified, then this truth function will not
                 perform class balancing.
 
                 Class balancing ensures that each class in a sound classification dataset
                 is represented equally (i.e., each class has the same number of augmented
-                targets). This is achieved by creating new class balancing augmentation
+                targets). This is achieved by creating new class balancing effect
                 rules and applying them to targets in underrepresented classes to create
-                more augmented targets for those classes.
+                more effected targets for those classes.
 
                 This rule must contain at least one random entry in order to guarantee
                 unique additional data.
 
-                See 'augmentations' for details on augmentation rules.
+                See 'effects' for details on effect rules.
 """ + get_truth_functions() + default
     # fmt: on
 
 
-def doc_augmentations() -> str:
+def doc_effects() -> str:
     # fmt: off
     return """
 Augmentation Rules
@@ -211,7 +211,7 @@ the list.
 If a value is specified using rand, then a randomized rule is generated
 dynamically per use.
 
-Rules may specify any or all of the following augmentations:
+Rules may specify any or all of the following effects:
 
  'normalize'    Normalize audio file to the specified level (in dBFS).
  'gain'         Apply an amplification or an attenuation to the audio signal.
@@ -237,16 +237,19 @@ Rules may specify any or all of the following augmentations:
                 'impulse_responses' parameter).
                 For targets, the impulse response is applied AFTER truth generation
                 and the resulting audio is still aligned with the truth. Random
-                syntax for 'ir' is simply 'rand' (i.e., do not specify <min> and <max>).
+                syntax for 'ir' is one of the following:
+                    'sai_choose()'              chooses a random IR from the entire list
+                    'sai_choose(<min>, <max>)'  chooses a random IR in the range <min> to <max>
+                    'sai_choose(<tag>)          chooses a random IR that matches <tag>
 
-Only the specified augmentations for a given rule are applied; all others are
+Only the specified effects for a given rule are applied; all others are
 skipped in the given rule. For example, if a rule only specifies 'tempo',
-then only a tempo augmentation is applied and all other possible augmentations
+then only a tempo effect is applied and all other possible effects
 are ignored (e.g., 'gain', 'pitch', etc.).
 
 Example:
 
-target_augmentations:
+target_effects:
   - normalize: -3.5
   - normalize: -3.5
     pitch: [-300, 300]
@@ -264,7 +267,7 @@ There are four rules given in this example.
 The first rule is simple:
   - normalize: -3.5
 
-This results in just one augmentation being applied to each target:
+This results in just one effect being applied to each target:
 
   normalize: -3.5
 
@@ -275,7 +278,7 @@ The second rule illustrates the use of lists to specify values:
     eq1: [[1000, 0.8, 3], [600, 1.0, -4], [800, 0.6, 0]]
 
 There are two values given for pitch, two for tempo, and three for EQ. This
-rule expands to 2 * 2 * 3 = 12 unique augmentations being applied to each
+rule expands to 2 * 2 * 3 = 12 unique effects being applied to each
 target:
 
   normalize: -3.5, pitch: -3, tempo: 0.8, eq1: [1000, 0.8,  3]
@@ -297,13 +300,13 @@ The third rule shows the use of rand:
     eq1: ["rand(100, 6000)", "rand(0.6, 1.0)", "rand(-6, 6)"]
     lpf: "rand(1000, 8000)"
 
-This rule is used to create randomized augmentations per use.
+This rule is used to create randomized effects per use.
 
 The fourth rule demonstrates the use of scalars, lists, and rand:
   - tempo: [0.9, 1, 1.1]
     eq1: [["rand(100, 7500)", 0.8, -10], ["rand(100, 7500)", 0.8, 10]]
 
-This rule expands to 6 unique augmentations being applied to each target
+This rule expands to 6 unique effects being applied to each target
 (list of 3 * list of 2). Here is the expansion:
 
   tempo: 0.9, eq1: ["rand(100, 7500)", 0.8, -10]
@@ -315,16 +318,16 @@ This rule expands to 6 unique augmentations being applied to each target
     # fmt: on
 
 
-def doc_target_augmentations() -> str:
+def doc_target_effects() -> str:
     import yaml
 
-    default = f"\nDefault value:\n\n{yaml.dump(get_default_config()['target_augmentations'])}"
+    default = f"\nDefault value:\n\n{yaml.dump(get_default_config()['target_effects'])}"
     # fmt: off
     return """
-'target_augmentations' is a mixture database configuration parameter that
-specifies a list of augmentation rules to use for each target.
+'target_effects' is a mixture database configuration parameter that
+specifies a list of effect rules to use for each target.
 
-See 'augmentations' for details on augmentation rules.
+See 'effects' for details on effect rules.
 """ + default
     # fmt: on
 
@@ -338,7 +341,7 @@ def doc_target_distortions() -> str:
 'target_distortions' is a mixture database configuration parameter that
 specifies a list of distortion rules to use for each target.
 
-See 'augmentations' for details on distortion rules.
+See 'effects' for details on distortion rules.
 """ + default
     # fmt: on
 
@@ -364,17 +367,17 @@ Required field:
     # fmt: on
 
 
-def doc_noise_augmentations() -> str:
+def doc_noise_effects() -> str:
     import yaml
 
-    default = f"\nDefault value:\n\n{yaml.dump(get_default_config()['noise_augmentations'])}"
+    default = f"\nDefault value:\n\n{yaml.dump(get_default_config()['noise_effects'])}"
 
     # fmt: off
     return """
-'noise_augmentations' is a mixture database configuration parameter that
-specifies a list of augmentation rules to use for each noise.
+'noise_effects' is a mixture database configuration parameter that
+specifies a list of effect rules to use for each noise.
 
-See 'augmentations' for details on augmentation rules.
+See 'effects' for details on effect rules.
 """ + default
     # fmt: on
 
@@ -386,7 +389,7 @@ def doc_snrs() -> str:
 'snrs' is a mixture database configuration parameter that specifies a list
 of required signal-to-noise ratios (in dB).
 
-All other augmentations are applied to both target and noise and then the
+All other effects are applied to both target and noise and then the
 energy levels are measured and the appropriate noise gain calculated to
 achieve the desired SNR.
 
@@ -407,7 +410,7 @@ list of random signal-to-noise ratios. The value(s) must be specified as
 random using the syntax: 'rand(<min>, <max>)'.
 
 Random SNRs behave slightly differently from regular or ordered SNRs. As with
-ordered SNRs, all other augmentations are applied to both target and noise and
+ordered SNRs, all other effects are applied to both target and noise and
 then the energy levels are measured and the appropriate noise gain calculated
 to achieve the desired SNR. However, unlike ordered SNRs, the desired SNR is
 randomized (per the given rule(s)) for each mixture, i.e., previous random
@@ -425,14 +428,14 @@ how to mix noises with targets.
 
 Supported modes:
 
- exhaustive          Use every noise/augmentation with every target/augmentation.
- non-exhaustive      Cycle through every target/augmentation without necessarily
-                     using all noise/augmentation combinations (reduced data set).
- non-combinatorial   Combine a target/augmentation with a single cut of a
-                     noise/augmentation non-exhaustively (each target/augmentation
-                     does not use each noise/augmentation). Cut has a random start
+ exhaustive          Use every noise/effect with every primary/effect.
+ non-exhaustive      Cycle through every primary/effect without necessarily
+                     using all noise/effect combinations (reduced data set).
+ non-combinatorial   Combine a primary/effect with a single cut of a
+                     noise/effect non-exhaustively (each primary/effect
+                     does not use each noise/effect). Cut has a random start
                      and loops back to the beginning if the end of a
-                     noise/augmentation is reached.
+                     noise/effect is reached.
 """ + default
     # fmt: on
 
@@ -444,7 +447,7 @@ def doc_impulse_responses() -> str:
 'impulse_responses' is a mixture database configuration parameter that specifies a
 list of impulse response files to use.
 
-See 'augmentations' for details.
+See 'effects' for details.
 """ + default
     # fmt: on
 
@@ -456,7 +459,7 @@ def doc_spectral_masks() -> str:
 'spectral_masks' is a mixture database configuration parameter that specifies
 a list of spectral mask rules.
 
-All other augmentations are applied including SNR and a mixture is generated
+All other effects are applied including SNR and a mixture is generated
 and then the spectral mask rules are applied to the resulting mixture feature.
 
 Rules must specify all the following parameters:

sonusai/lsdb.py

@@ -81,7 +81,7 @@ def lsdb(
     mixids = mixdb.mixids_to_list(mixids)
 
     if len(mixids) == 1:
-        print_mixture_details(mixdb=mixdb, mixid=mixids[0], desc_len=desc_len, print_fn=logger.info)
+        print_mixture_details(mixdb=mixdb, mixid=mixids[0], print_fn=logger.info)
         if all_class_counts:
             # TODO: fix class count
             logger.info("All class count not supported")

sonusai/mixture/__init__.py

@@ -28,4 +28,4 @@ from .helpers import inverse_transform
 from .helpers import write_mixture_metadata
 from .log_duration_and_sizes import log_duration_and_sizes
 from .mixdb import MixtureDatabase
-from .mixdb import db_file
+from .db_file import db_file

sonusai/mixture/db.py

@@ -0,0 +1,163 @@
+import contextlib
+import sqlite3
+from os import remove
+from os.path import exists
+from sqlite3 import Connection
+from sqlite3 import Cursor
+from typing import Any
+
+from .. import logger_db
+from .db_file import db_file
+
+
+class SQLiteDatabase:
+    """A context manager for SQLite database connections with configurable behavior."""
+
+    # Constants for database configuration
+    READONLY_MODE = "?mode=ro"
+    WRITE_OPTIMIZED_PRAGMAS = (
+        "?_journal_mode=OFF&_synchronous=OFF&_cache_size=10000&_temp_store=MEMORY&_locking_mode=EXCLUSIVE"
+    )
+    CONNECTION_TIMEOUT = 20
+
+    def __init__(
+        self,
+        location: str,
+        create: bool = False,
+        readonly: bool = True,
+        test: bool = False,
+        verbose: bool = False,
+    ) -> None:
+        """Initialize SQLite database connection manager.
+
+        Args:
+            location: Path to the database file.
+            create: If True, create a new database file, overwriting any existing one.
+            readonly: If True, open the database in read-only mode.
+            test: If True, use the test database path.
+            verbose: If True, enable SQL statement logging.
+        """
+        self.location = location
+        self.create = create
+        self.readonly = readonly
+        self.test = test
+        self.verbose = verbose
+        self.con: Connection | None = None
+        self.cur: Cursor | None = None
+
+    def __enter__(self) -> Cursor:
+        """Enter the context manager, establishing the database connection.
+
+        Returns:
+            A database cursor for executing queries.
+
+        Raises:
+            sqlite3.Error: If connection fails.
+        """
+        try:
+            self._establish_connection()
+        except Exception:
+            self._close_resources()
+            raise
+
+        if self.cur:
+            return self.cur
+        raise sqlite3.Error("Failed to connect to database")
+
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_val: BaseException | None,
+        exc_tb: Any | None,
+    ) -> None:
+        """Exit the context manager, committing changes if appropriate and closing resources.
+
+        Args:
+            exc_type: The exception type, if any.
+            exc_val: The exception value, if any.
+            exc_tb: The exception traceback, if any.
+        """
+        if self.con:
+            if exc_type is None and not self.readonly:
+                # Commit only on successful exit if not readonly
+                self.con.commit()
+            self._close_resources()
+
+    def _close_resources(self) -> None:
+        """Safely close database cursor and connection resources."""
+        if self.cur:
+            with contextlib.suppress(sqlite3.Error):
+                self.cur.close()
+            self.cur = None
+
+        if self.con:
+            with contextlib.suppress(sqlite3.Error):
+                self.con.close()
+            self.con = None
+
+    def _establish_connection(self) -> None:
+        """Establish a connection to the SQLite database.
+
+        Raises:
+            OSError: If the database file doesn't exist and create=False.
+            sqlite3.Error: If connection to the database fails.
+        """
+        db_path = self._get_db_path()
+        self._prepare_db_file(db_path)
+        uri = self._build_connection_uri(db_path)
+
+        try:
+            self.con = sqlite3.connect(f"file:{uri}", uri=True, timeout=self.CONNECTION_TIMEOUT)
+            if self.verbose and self.con:
+                self.con.set_trace_callback(logger_db.debug)
+        except sqlite3.Error as e:
+            raise sqlite3.Error(f"Failed to connect to database: {e}") from e
+
+        self.cur = self.con.cursor()
+
+    def _get_db_path(self) -> str:
+        """Get the database file path based on location and test settings.
+
+        Returns:
+            The path to the database file.
+        """
+        return db_file(self.location, self.test)
+
+    def _prepare_db_file(self, db_path: str) -> None:
+        """Prepare the database file based on creation settings.
+
+        Args:
+            db_path: Path to the database file.
+
+        Raises:
+            OSError: If the database file doesn't exist and create=False.
+        """
+        if self.create and exists(db_path):
+            remove(db_path)
+
+        if not self.create and not exists(db_path):
+            raise OSError(f"Could not find mixture database in {self.location}")
+
+    def _build_connection_uri(self, db_path: str) -> str:
+        """Build the SQLite connection URI with appropriate options.
+
+        Args:
+            db_path: Path to the database file.
+
+        Returns:
+            A properly formatted SQLite URI with appropriate options.
+        """
+        uri = db_path
+
+        # Add readonly mode if needed
+        if not self.create and self.readonly:
+            uri += self.READONLY_MODE
+
+        # Add optimized pragmas for write mode
+        if not self.readonly:
+            if "?" in uri:
+                uri = uri.replace("?", f"{self.WRITE_OPTIMIZED_PRAGMAS}&")
+            else:
+                uri += self.WRITE_OPTIMIZED_PRAGMAS
+
+        return uri

sonusai/mixture/db_file.py

@@ -0,0 +1,10 @@
+from os.path import join
+from os.path import normpath
+
+from .constants import MIXDB_NAME
+from .constants import TEST_MIXDB_NAME
+
+
+def db_file(location: str, test: bool = False) -> str:
+    name = TEST_MIXDB_NAME if test else MIXDB_NAME
+    return normpath(join(location, name))

sonusai/mixture/effects.py

@@ -1,8 +1,5 @@
-# sai_rand
-# sai_rand_choice
-# sai_rand_choice_nr
-# sai_sequence
-# sai_expand
+from numpy.ma.core import choose
+from urllib3.filepost import choose_boundary
 
 from ..datatypes import AudioT
 from ..datatypes import Effects
@@ -28,40 +25,11 @@ def get_effect_rules(location: str, config: dict, test: bool = False) -> dict[st
     return rules
 
 
-def sai_expand(text: str) -> list[str]:
-    import re
-
-    # search pattern
-    pattern = re.compile(r"sai_expand\((.+?)\)")
-
-    # initialize with input
-    expanded = [text]
-
-    # look for pattern
-    result = re.search(pattern, text)
-
-    # if found
-    if result:
-        # remove entry we are expanding
-        expanded.pop()
-
-        # convert match into list stripped of whitespace
-        values = result.group(1).replace(" ", "").split(",")
-
-        # loop over values
-        for value in values:
-            # replace pattern with value
-            replacement = re.sub(pattern, value, text, count=1)
-
-            # extend result with expand of replacement (for handling multiple expands in a single rule)
-            expanded.extend(sai_expand(replacement))
-
-    return expanded
-
-
 def _expand_effect_rules(expanded_rules: list[dict], rule: dict) -> list[dict]:
     from copy import deepcopy
 
+    from .parse import sai_expand
+
     for key in ("pre", "post"):
         if key in rule:
             value = rule[key]
@@ -274,8 +242,8 @@ def evaluate_sai_random_float(text: str) -> str:
     return resolved
 
 
-def evaluate_sai_random_ir(mixdb: MixtureDatabase, text: str) -> str:
-    """Evaluate 'sai_rand' directive for ir
+def evaluate_sai_choose_ir(mixdb: MixtureDatabase, text: str) -> str:
+    """Evaluate 'sai_choose' directive for ir
 
     :param mixdb: Mixture database
     :param text: Text to evaluate
@@ -285,11 +253,11 @@ def evaluate_sai_random_ir(mixdb: MixtureDatabase, text: str) -> str:
     from random import choice
     from random import randint
 
-    rand_pattern = re.compile(r"^ir sai_rand$")
-    rand_range_pattern = re.compile(r"^ir sai_rand\(([-+]?\d+),\s*([-+]?\d+)\)$")
-    rand_tag_pattern = re.compile(r"^ir sai_rand\((\w+)\)$")
+    choose_pattern = re.compile(r"^ir sai_choose\(\)$")
+    choose_range_pattern = re.compile(r"^ir sai_choose\(([-+]?\d+),\s*([-+]?\d+)\)$")
+    choose_tag_pattern = re.compile(r"^ir sai_choose\((\w+)\)$")
 
-    def rand_range_repl(m) -> str:
+    def choose_range_repl(m) -> str:
         lower = int(m.group(1))
         upper = int(m.group(2))
         if (
@@ -304,22 +272,22 @@ def evaluate_sai_random_ir(mixdb: MixtureDatabase, text: str) -> str:
             raise ValueError(f"Invalid rule: '{text}'. Values must be integers between 0 and {mixdb.num_ir_files - 1}.")
         return f"ir {randint(lower, upper)}"  # noqa: S311
 
-    def rand_tag_repl(m) -> str:
+    def choose_tag_repl(m) -> str:
         return m.group(1)
 
-    if re.match(rand_pattern, text):
+    if re.match(choose_pattern, text):
         return f"ir {randint(0, mixdb.num_ir_files - 1)}"  # noqa: S311
 
-    if re.match(rand_range_pattern, text):
+    if re.match(choose_range_pattern, text):
         try:
-            return f"ir {eval(re.sub(rand_range_pattern, rand_range_repl, text))}"  # noqa: S307
+            return f"ir {eval(re.sub(choose_range_pattern, choose_range_repl, text))}"  # noqa: S307
         except Exception as e:
             raise ValueError(
                 f"Invalid rule: '{text}'. Values must be integers between 0 and {mixdb.num_ir_files - 1}."
             ) from e
 
-    if re.match(rand_tag_pattern, text):
-        tag = re.sub(rand_tag_pattern, rand_tag_repl, text)
+    if re.match(choose_tag_pattern, text):
+        tag = re.sub(choose_tag_pattern, choose_tag_repl, text)
         if tag in mixdb.ir_tags:
             return f"ir {choice(mixdb.ir_file_ids_for_tag(tag))}"  # noqa: S311
 
@@ -336,10 +304,9 @@ def effects_from_rules(mixdb: MixtureDatabase, rules: Effects) -> Effects:
         entries = getattr(effects, key)
         for idx, entry in enumerate(entries):
             if entry.find("sai_rand") != -1:
-                if entry.startswith("ir"):
-                    entries[idx] = evaluate_sai_random_ir(mixdb, entry)
-                else:
-                    entries[idx] = evaluate_sai_random_float(entry)
+                entries[idx] = evaluate_sai_random_float(entry)
+            if entry.startswith("ir sai_choose"):
+                entries[idx] = evaluate_sai_choose_ir(mixdb, entry)
         setattr(effects, key, entries)
 
     return effects