sonusai 0.18.0__py3-none-any.whl → 0.18.1__py3-none-any.whl

sonusai/mixture/db_datatypes.py

@@ -0,0 +1,72 @@
+from collections import namedtuple
+
+TruthSettingRecord = namedtuple('TruthSettingRecord', [
+    'id',
+    'setting'])
+
+TargetFileRecord = namedtuple('TargetFileRecord', [
+    'id',
+    'name',
+    'samples',
+    'level_type',
+    'speaker_id'])
+
+NoiseFileRecord = namedtuple('NoiseFileRecord', [
+    'id',
+    'name',
+    'samples'])
+
+TopRecord = namedtuple('TopRecord', [
+    'id',
+    'version',
+    'class_balancing',
+    'feature',
+    'noise_mix_mode',
+    'num_classes',
+    'seed',
+    'truth_mutex',
+    'truth_reduction_function',
+    'mixid_width',
+    'speaker_metadata_tiers',
+    'textgrid_metadata_tiers'])
+
+ClassLabelRecord = namedtuple('ClassLabelRecord', [
+    'id',
+    'label'])
+
+ClassWeightsThresholdRecord = namedtuple('ClassWeightsThresholdRecord', [
+    'id',
+    'threshold'])
+
+ImpulseResponseFileRecord = namedtuple('ImpulseResponseFileRecord', [
+    'id',
+    'file'])
+
+SpectralMaskRecord = namedtuple('SpectralMaskRecord', [
+    'id',
+    'f_max_width',
+    'f_num',
+    't_max_width',
+    't_num',
+    't_max_percent'])
+
+TargetRecord = namedtuple('TargetRecord', [
+    'id',
+    'file_id',
+    'augmentation',
+    'gain'])
+
+MixtureRecord = namedtuple('MixtureRecord', [
+    'id',
+    'name',
+    'noise_file_id',
+    'noise_augmentation',
+    'noise_offset',
+    'noise_snr_gain',
+    'random_snr',
+    'snr',
+    'samples',
+    'spectral_mask_id',
+    'spectral_mask_seed',
+    'target_snr_gain'
+])

sonusai/mixture/generation.py

@@ -992,6 +992,9 @@ def _populate_speaker_table(location: str, target_files: TargetFiles, test: bool
              description[0] not in ('id', 'parent')]
     con.execute("UPDATE top SET speaker_metadata_tiers=? WHERE top.id = ?", (json.dumps(tiers), 1))
 
+    if 'speaker_id' in tiers:
+        con.execute("CREATE INDEX speaker_speaker_id_idx ON speaker (speaker_id)")
+
     con.commit()
     con.close()
 

sonusai/mixture/helpers.py

@@ -26,6 +26,8 @@ from sonusai.mixture.datatypes import TargetFiles
 from sonusai.mixture.datatypes import Targets
 from sonusai.mixture.datatypes import TransformConfig
 from sonusai.mixture.datatypes import Truth
+from sonusai.mixture.db_datatypes import MixtureRecord
+from sonusai.mixture.db_datatypes import TargetRecord
 from sonusai.mixture.mixdb import MixtureDatabase
 
 
@@ -228,7 +230,7 @@ def from_mixture(mixture: Mixture) -> tuple[str, int, str, int, float, bool, flo
             mixture.target_snr_gain)
 
 
-def to_mixture(entry: tuple[str, int, str, int, float, bool, float, int, int, int, float], targets: Targets) -> Mixture:
+def to_mixture(entry: MixtureRecord, targets: Targets) -> Mixture:
     import json
 
     from sonusai.utils import dataclass_from_dict
@@ -238,32 +240,32 @@ def to_mixture(entry: tuple[str, int, str, int, float, bool, float, int, int, in
     from .datatypes import UniversalSNR
 
     return Mixture(targets=targets,
-                   name=entry[0],
-                   noise=Noise(file_id=entry[1],
-                               augmentation=dataclass_from_dict(Augmentation, json.loads(entry[2])),
-                               offset=entry[3]),
-                   noise_snr_gain=entry[4],
-                   snr=UniversalSNR(is_random=entry[5], value=entry[6]),
-                   samples=entry[7],
-                   spectral_mask_id=entry[8],
-                   spectral_mask_seed=entry[9],
-                   target_snr_gain=entry[10])
+                   name=entry.name,
+                   noise=Noise(file_id=entry.noise_file_id,
+                               augmentation=dataclass_from_dict(Augmentation, json.loads(entry.noise_augmentation)),
+                               offset=entry.noise_offset),
+                   noise_snr_gain=entry.noise_snr_gain,
+                   snr=UniversalSNR(is_random=entry.random_snr, value=entry.snr),
+                   samples=entry.samples,
+                   spectral_mask_id=entry.spectral_mask_id,
+                   spectral_mask_seed=entry.spectral_mask_seed,
+                   target_snr_gain=entry.target_snr_gain)
 
 
 def from_target(target: Target) -> tuple[int, str, float]:
     return target.file_id, target.augmentation.to_json(), target.gain
 
 
-def to_target(entry: tuple[int, str, float]) -> Target:
+def to_target(entry: TargetRecord) -> Target:
     import json
 
     from sonusai.utils import dataclass_from_dict
     from .datatypes import Augmentation
     from .datatypes import Target
 
-    return Target(file_id=entry[0],
-                  augmentation=dataclass_from_dict(Augmentation, json.loads(entry[1])),
-                  gain=entry[2])
+    return Target(file_id=entry.file_id,
+                  augmentation=dataclass_from_dict(Augmentation, json.loads(entry.augmentation)),
+                  gain=entry.gain)
 
 
 def read_mixture_data(name: str, items: list[str] | str) -> Any:

sonusai/mixture/mixdb.py

@@ -129,32 +129,32 @@ class MixtureDatabase:
     @cached_property
     def num_classes(self) -> int:
         with self.db() as c:
-            return int(c.execute("SELECT top.num_classes from top").fetchone()[0])
+            return int(c.execute("SELECT top.num_classes FROM top").fetchone()[0])
 
     @cached_property
     def truth_mutex(self) -> bool:
         with self.db() as c:
-            return bool(c.execute("SELECT top.truth_mutex from top").fetchone()[0])
+            return bool(c.execute("SELECT top.truth_mutex FROM top").fetchone()[0])
 
     @cached_property
     def truth_reduction_function(self) -> str:
         with self.db() as c:
-            return str(c.execute("SELECT top.truth_reduction_function from top").fetchone()[0])
+            return str(c.execute("SELECT top.truth_reduction_function FROM top").fetchone()[0])
 
     @cached_property
     def noise_mix_mode(self) -> str:
         with self.db() as c:
-            return str(c.execute("SELECT top.noise_mix_mode from top").fetchone()[0])
+            return str(c.execute("SELECT top.noise_mix_mode FROM top").fetchone()[0])
 
     @cached_property
     def class_balancing(self) -> bool:
         with self.db() as c:
-            return bool(c.execute("SELECT top.class_balancing from top").fetchone()[0])
+            return bool(c.execute("SELECT top.class_balancing FROM top").fetchone()[0])
 
     @cached_property
     def feature(self) -> str:
         with self.db() as c:
-            return str(c.execute("SELECT top.feature from top").fetchone()[0])
+            return str(c.execute("SELECT top.feature FROM top").fetchone()[0])
 
     @cached_property
     def fg_decimation(self) -> int:
@@ -282,14 +282,16 @@ class MixtureDatabase:
 
         :return: Spectral masks
         """
+        from .db_datatypes import SpectralMaskRecord
+
         with self.db() as c:
-            results = c.execute(
-                "SELECT spectral_mask.f_max_width, f_num, t_max_width, t_num, t_max_percent FROM spectral_mask")
-            return [SpectralMask(f_max_width=spectral_mask[0],
-                                 f_num=spectral_mask[1],
-                                 t_max_width=spectral_mask[2],
-                                 t_num=spectral_mask[3],
-                                 t_max_percent=spectral_mask[4]) for spectral_mask in results.fetchall()]
+            spectral_masks = [SpectralMaskRecord(*result) for result in
+                              c.execute("SELECT * FROM spectral_mask").fetchall()]
+            return [SpectralMask(f_max_width=spectral_mask.f_max_width,
+                                 f_num=spectral_mask.f_num,
+                                 t_max_width=spectral_mask.t_max_width,
+                                 t_num=spectral_mask.t_num,
+                                 t_max_percent=spectral_mask.t_max_percent) for spectral_mask in spectral_masks]
 
     def spectral_mask(self, sm_id: int) -> SpectralMask:
         """Get spectral mask with ID from db
@@ -309,27 +311,29 @@ class MixtureDatabase:
 
         from .datatypes import TruthSetting
         from .datatypes import TruthSettings
+        from .db_datatypes import TargetFileRecord
 
         with self.db() as c:
             target_files: TargetFiles = []
-            for target in c.execute(
-                    "SELECT target_file.name, samples, level_type, id, speaker_id FROM target_file").fetchall():
+            target_file_records = [TargetFileRecord(*result) for result in
+                                   c.execute("SELECT * FROM target_file").fetchall()]
+            for target_file_record in target_file_records:
                 truth_settings: TruthSettings = []
-                for ts in c.execute(
+                for truth_setting_records in c.execute(
                         "SELECT truth_setting.setting " +
                         "FROM truth_setting, target_file_truth_setting " +
                         "WHERE ? = target_file_truth_setting.target_file_id " +
                         "AND truth_setting.id = target_file_truth_setting.truth_setting_id",
-                        (target[3],)).fetchall():
-                    entry = json.loads(ts[0])
-                    truth_settings.append(TruthSetting(config=entry.get('config', None),
-                                                       function=entry.get('function', None),
-                                                       index=entry.get('index', None)))
-                target_files.append(TargetFile(name=target[0],
-                                               samples=target[1],
-                                               level_type=target[2],
+                        (target_file_record.id,)).fetchall():
+                    truth_setting = json.loads(truth_setting_records[0])
+                    truth_settings.append(TruthSetting(config=truth_setting.get('config', None),
+                                                       function=truth_setting.get('function', None),
+                                                       index=truth_setting.get('index', None)))
+                target_files.append(TargetFile(name=target_file_record.name,
+                                               samples=target_file_record.samples,
+                                               level_type=target_file_record.level_type,
                                                truth_settings=truth_settings,
-                                               speaker_id=target[4]))
+                                               speaker_id=target_file_record.speaker_id))
             return target_files
 
     @cached_property
@@ -439,18 +443,16 @@ class MixtureDatabase:
         """
         from .helpers import to_mixture
         from .helpers import to_target
+        from .db_datatypes import MixtureRecord
+        from .db_datatypes import TargetRecord
 
         with self.db() as c:
             mixtures: Mixtures = []
-            for mixture in c.execute(
-                    "SELECT mixture.name, noise_file_id, noise_augmentation, noise_offset, noise_snr_gain, " +
-                    "random_snr, snr, samples, spectral_mask_id, spectral_mask_seed, target_snr_gain, id " +
-                    "FROM mixture").fetchall():
-                targets = [to_target(target) for target in c.execute(
-                    "SELECT target.file_id, augmentation, gain " +
-                    "FROM target, mixture_target " +
+            for mixture in [MixtureRecord(*record) for record in c.execute("SELECT * FROM mixture").fetchall()]:
+                targets = [to_target(TargetRecord(*target)) for target in c.execute(
+                    "SELECT target.* FROM target, mixture_target " +
                     "WHERE ? = mixture_target.mixture_id AND target.id = mixture_target.target_id",
-                    (mixture[11],)).fetchall()]
+                    (mixture.id,)).fetchall()]
                 mixtures.append(to_mixture(mixture, targets))
             return mixtures
 
@@ -474,7 +476,7 @@ class MixtureDatabase:
     @cached_property
     def mixid_width(self) -> int:
         with self.db() as c:
-            return int(c.execute("SELECT top.mixid_width from top").fetchone()[0])
+            return int(c.execute("SELECT top.mixid_width FROM top").fetchone()[0])
 
     def location_filename(self, name: str) -> str:
         """Add the location to the given file name
@@ -1001,17 +1003,8 @@ class MixtureDatabase:
     def speech_metadata_tiers(self) -> list[str]:
         return sorted(set(self.speaker_metadata_tiers + self.textgrid_metadata_tiers))
 
-    def speaker(self, speaker_id: int | None, tier: str) -> Optional[str]:
-        if speaker_id is None:
-            return None
-
-        with self.db() as c:
-            data = c.execute(f'SELECT {tier} FROM speaker WHERE ? = id', (speaker_id,)).fetchone()
-            if data is None:
-                return None
-            if data[0] is None:
-                return None
-            return data[0]
+    def speaker(self, s_id: int | None, tier: str) -> Optional[str]:
+        return _speaker(self.db, s_id, tier)
 
     def speech_metadata(self, tier: str) -> list[str]:
         from .helpers import get_textgrid_tier_from_target_file
@@ -1096,27 +1089,22 @@ class MixtureDatabase:
                 return x == value
 
         # First get list of matching target files
-        target_files: list[str] = []
+        target_file_ids: list[int] = []
         is_textgrid = tier in self.textgrid_metadata_tiers
-        for target_file in self.target_files:
+        for target_file_id, target_file in enumerate(self.target_files):
             if is_textgrid:
                 metadata = get_textgrid_tier_from_target_file(target_file.name, tier)
             else:
                 metadata = self.speaker(target_file.speaker_id, tier)
 
             if not isinstance(metadata, list) and predicate(metadata):
-                target_files.append(target_file.name)
+                target_file_ids.append(target_file_id + 1)
 
         # Next get list of mixture IDs that contain those target files
-        m_ids: list[int] = []
-        for m_id in self.mixids_to_list():
-            mixid_target_files = [self.target_file(target.file_id).name for target in self.mixture(m_id).targets]
-            for mixid_target_file in mixid_target_files:
-                if mixid_target_file in target_files:
-                    m_ids.append(m_id)
-
-        # Return sorted, unique list of mixture IDs
-        return sorted(list(set(m_ids)))
+        with self.db() as c:
+            m_ids = c.execute("SELECT mixture_id FROM mixture_target " +
+                               f"WHERE mixture_target.target_id IN ({','.join(map(str, target_file_ids))})").fetchall()
+        return [x[0] - 1 for x in m_ids]
 
     def mixture_all_speech_metadata(self, m_id: int) -> list[dict[str, SpeechMetadata]]:
         from .helpers import mixture_all_speech_metadata
@@ -1300,17 +1288,16 @@ def _spectral_mask(db: partial, sm_id: int) -> SpectralMask:
     :param sm_id: Spectral mask ID
     :return: Spectral mask
     """
+    from .db_datatypes import SpectralMaskRecord
+
     with db() as c:
-        spectral_mask = c.execute(
-            "SELECT spectral_mask.f_max_width, f_num, t_max_width, t_num, t_max_percent " +
-            "FROM spectral_mask " +
-            "WHERE ? = spectral_mask.id",
-            (sm_id,)).fetchone()
-        return SpectralMask(f_max_width=spectral_mask[0],
-                            f_num=spectral_mask[1],
-                            t_max_width=spectral_mask[2],
-                            t_num=spectral_mask[3],
-                            t_max_percent=spectral_mask[4])
+        spectral_mask = SpectralMaskRecord(*c.execute("SELECT * FROM spectral_mask WHERE ? = spectral_mask.id",
+                                                      (sm_id,)).fetchone())
+        return SpectralMask(f_max_width=spectral_mask.f_max_width,
+                            f_num=spectral_mask.f_num,
+                            t_max_width=spectral_mask.t_max_width,
+                            t_num=spectral_mask.t_num,
+                            t_max_percent=spectral_mask.t_max_percent)
 
 
 @lru_cache
@@ -1325,11 +1312,11 @@ def _target_file(db: partial, t_id: int) -> TargetFile:
 
     from .datatypes import TruthSetting
     from .datatypes import TruthSettings
+    from .db_datatypes import TargetFileRecord
 
     with db() as c:
-        target = c.execute(
-            "SELECT target_file.name, samples, level_type, speaker_id FROM target_file WHERE ? = target_file.id",
-            (t_id,)).fetchone()
+        target_file = TargetFileRecord(
+            *c.execute("SELECT * FROM target_file WHERE ? = target_file.id", (t_id,)).fetchone())
 
         truth_settings: TruthSettings = []
         for ts in c.execute(
@@ -1342,11 +1329,11 @@ def _target_file(db: partial, t_id: int) -> TargetFile:
             truth_settings.append(TruthSetting(config=entry.get('config', None),
                                                function=entry.get('function', None),
                                                index=entry.get('index', None)))
-        return TargetFile(name=target[0],
-                          samples=target[1],
-                          level_type=target[2],
+        return TargetFile(name=target_file.name,
+                          samples=target_file.samples,
+                          level_type=target_file.level_type,
                           truth_settings=truth_settings,
-                          speaker_id=target[3])
+                          speaker_id=target_file.speaker_id)
 
 
 @lru_cache
@@ -1387,19 +1374,29 @@ def _mixture(db: partial, m_id: int) -> Mixture:
     """
     from .helpers import to_mixture
     from .helpers import to_target
+    from .db_datatypes import MixtureRecord
+    from .db_datatypes import TargetRecord
 
     with db() as c:
-        mixture = c.execute(
-            "SELECT mixture.name, noise_file_id, noise_augmentation, noise_offset, noise_snr_gain, " +
-            "random_snr, snr, samples, spectral_mask_id, spectral_mask_seed, target_snr_gain, id " +
-            "FROM mixture " +
-            "WHERE ? = mixture.id",
-            (m_id + 1,)).fetchone()
-
-        targets = [to_target(target) for target in c.execute(
-            "SELECT target.file_id, augmentation, gain " +
+        mixture = MixtureRecord(*c.execute("SELECT * FROM mixture WHERE ? = mixture.id", (m_id + 1,)).fetchone())
+        targets = [to_target(TargetRecord(*target)) for target in c.execute(
+            "SELECT target.* " +
             "FROM target, mixture_target " +
             "WHERE ? = mixture_target.mixture_id AND target.id = mixture_target.target_id",
-            (mixture[11],)).fetchall()]
+            (mixture.id,)).fetchall()]
 
         return to_mixture(mixture, targets)
+
+
+@lru_cache
+def _speaker(db: partial, s_id: int | None, tier: str) -> Optional[str]:
+    if s_id is None:
+        return None
+
+    with db() as c:
+        data = c.execute(f'SELECT {tier} FROM speaker WHERE ? = id', (s_id,)).fetchone()
+        if data is None:
+            return None
+        if data[0] is None:
+            return None
+        return data[0]

{sonusai-0.18.0.dist-info → sonusai-0.18.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sonusai
-Version: 0.18.0
+Version: 0.18.1
 Summary: Framework for building deep neural network models for sound, speech, and voice AI
 Home-page: https://aaware.com
 License: GPL-3.0-only

{sonusai-0.18.0.dist-info → sonusai-0.18.1.dist-info}/RECORD

@@ -36,13 +36,14 @@ sonusai/mixture/class_count.py,sha256=_wFnVl2yEOnbor7pLg7cYOUeX6nioov-03Cv3SEbh2
 sonusai/mixture/config.py,sha256=d2IzZ1samHWGMpoKzSmUwMyAWWhgmyNoxyO8oiUwbsg,22193
 sonusai/mixture/constants.py,sha256=ZRM7Z8e6EwnL9RwaMVxks-QokN9KMWxnQzAf9VNxa9M,1408
 sonusai/mixture/datatypes.py,sha256=uVJtT2sVGS531pSglhaLLF5hZcI3_0oKQOWmMkrCwqo,8253
+sonusai/mixture/db_datatypes.py,sha256=GDYbcSrlgUJsesiUUNnR4s5aBkMgviiNSQDaBcgYX7I,1428
 sonusai/mixture/eq_rule_is_valid.py,sha256=MpQwRA5M76wSiQWEI1lW2cLFdPaMttBLcQp3tWD8efM,1243
 sonusai/mixture/feature.py,sha256=Rwuf82IoXzhHPGbKYVGcatImF_ssBf_FfvbqghVPXtg,4116
-sonusai/mixture/generation.py,sha256=H_9kgresvNAEI6pmqAEwOaxQMPg6geVw6G0Y8fmbu_o,42580
-sonusai/mixture/helpers.py,sha256=4gb_t65Aw9O8hUgDzq-7D_t6LynfQDl1jcuFhQIAdWI,24590
+sonusai/mixture/generation.py,sha256=ohZnhtHIrdQDql2OF703NnhK07Ys-1qAjiwrIql-oMw,42694
+sonusai/mixture/helpers.py,sha256=eC9ZysEa-83VLKen_9PKWzr8w9dkHj4lp6rMB2fNLbg,24759
 sonusai/mixture/log_duration_and_sizes.py,sha256=baTUpqyM15wA125jo9E3posmVJUe3WlpksyO6v9Jul0,1347
 sonusai/mixture/mapped_snr_f.py,sha256=Fdf2uw62FvyKvVy5VywaUtPZGO1zCWQsHlte0bwkKPQ,3121
-sonusai/mixture/mixdb.py,sha256=xgHD43OgXaFKb1o4nsCPt-bd_mnk-PPetX4OGCGZ8DM,51582
+sonusai/mixture/mixdb.py,sha256=XSJQKYaUfQylpWbGjfxfP7dendr-9JFcmLWNyE0qUwQ,51697
 sonusai/mixture/soundfile_audio.py,sha256=mHa5SIXsu_uE0j3DO52GydRJrvWSzU_nII-7YJfQ6Qo,4154
 sonusai/mixture/sox_audio.py,sha256=HT3kYA9TP5QPCuoOJdUMnGVN-qY6q96DGL8zxuog76o,12277
 sonusai/mixture/sox_augmentation.py,sha256=kBWPrsFk0EBi71nLcKt5v0GA34bY7g9D9x0cEamNWbU,4564
@@ -115,7 +116,7 @@ sonusai/utils/stratified_shuffle_split.py,sha256=rJNXvBp-GxoKzH3OpL7k0ANSu5xMP2z
 sonusai/utils/write_audio.py,sha256=ZsPGExwM86QHLLN2LOWekK2uAqf5pV_1oRW811p0QAI,840
 sonusai/utils/yes_or_no.py,sha256=eMLXBVH0cEahiXY4W2KNORmwNQ-ba10eRtldh0y4NYg,263
 sonusai/vars.py,sha256=m2AefF0m5bXWGXpJj8Pi42zWL2ydeEj7bkak3GrtMyM,940
-sonusai-0.18.0.dist-info/METADATA,sha256=LwOGcp1V_87ef8oyjI8Kjwo5K8tbIKjpRfJ17dW0Dbc,2591
-sonusai-0.18.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-sonusai-0.18.0.dist-info/entry_points.txt,sha256=zMNjEphEPO6B3cD1GNpit7z-yA9tUU5-j3W2v-UWstU,92
-sonusai-0.18.0.dist-info/RECORD,,
+sonusai-0.18.1.dist-info/METADATA,sha256=cLhSYAb5FSCHlrBCG0xA2hwa_4S86JUEWIqY85xA5o8,2591
+sonusai-0.18.1.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+sonusai-0.18.1.dist-info/entry_points.txt,sha256=zMNjEphEPO6B3cD1GNpit7z-yA9tUU5-j3W2v-UWstU,92
+sonusai-0.18.1.dist-info/RECORD,,