sonusai 1.0.4__py3-none-any.whl → 1.0.6__py3-none-any.whl

sonusai/__init__.py

@@ -26,19 +26,31 @@ commands_doc = """
 logger = logging.getLogger("sonusai")
 logger.setLevel(logging.DEBUG)
 formatter = logging.Formatter("%(message)s")
+formatter_db = logging.Formatter("%(asctime)s %(message)s")
 console_handler = RichHandler(show_level=False, show_path=False, show_time=False)
 console_handler.setLevel(logging.DEBUG)
 console_handler.setFormatter(formatter)
 logger.addHandler(console_handler)
 
+logger_db = logging.getLogger("sonusai_db")
+logger_db.setLevel(logging.DEBUG)
 
 # create file handler
 def create_file_handler(filename: str) -> None:
+    from pathlib import Path
+
     fh = logging.FileHandler(filename=filename, mode="w")
     fh.setLevel(logging.DEBUG)
     fh.setFormatter(formatter)
     logger.addHandler(fh)
 
+    filename_db = Path(filename)
+    filename_db = filename_db.parent / (filename_db.stem + "_dbtrace" + filename_db.suffix)
+    fh = logging.FileHandler(filename=filename_db, mode="w")
+    fh.setLevel(logging.DEBUG)
+    fh.setFormatter(formatter_db)
+    logger_db.addHandler(fh)
+
 
 # update console handler
 def update_console_handler(verbose: bool) -> None:

sonusai/genmixdb.py

@@ -30,6 +30,7 @@ def genmixdb(
     logging: bool = True,
     show_progress: bool = False,
     test: bool = False,
+    verbose: bool = False,
     save_json: bool = False,
     no_par: bool = False,
 ) -> None:
@@ -69,11 +70,11 @@ def genmixdb(
 
     mixdb = MixtureDatabase(location, test)
 
-    populate_top_table(location, config, test)
-    populate_class_label_table(location, config, test)
-    populate_class_weights_threshold_table(location, config, test)
-    populate_spectral_mask_table(location, config, test)
-    populate_truth_parameters_table(location, config, test)
+    populate_top_table(location, config, test, verbose)
+    populate_class_label_table(location, config, test, verbose)
+    populate_class_weights_threshold_table(location, config, test, verbose)
+    populate_spectral_mask_table(location, config, test, verbose)
+    populate_truth_parameters_table(location, config, test, verbose)
 
     seed(config["seed"])
 
@@ -94,7 +95,7 @@ def genmixdb(
     if logging:
         logger.info("Populating source file table")
 
-    populate_source_file_table(location, source_files, test)
+    populate_source_file_table(location, source_files, test, verbose)
 
     if logging:
         logger.info("Sources summary")
@@ -129,7 +130,7 @@ def genmixdb(
     if logging:
         logger.info("Populating impulse response file table")
 
-    populate_impulse_response_file_table(location, ir_files, test)
+    populate_impulse_response_file_table(location, ir_files, test, verbose)
 
     if logging:
         logger.debug("List of impulse responses:")
@@ -219,6 +220,7 @@ def genmixdb(
         location=location,
         mixtures=mixtures,
         test=test,
+        verbose=verbose,
         logging=logging,
         show_progress=show_progress,
     )
@@ -330,6 +332,7 @@ def main() -> None:
         save_mix=save_mix,
         show_progress=True,
         save_json=save_json,
+        verbose=verbose,
         no_par=no_par,
     )
 

sonusai/mixture/generation.py

@@ -17,10 +17,10 @@ def config_file(location: str) -> str:
     return join(location, "config.yml")
 
 
-def initialize_db(location: str, test: bool = False) -> None:
+def initialize_db(location: str, test: bool = False, verbose: bool = False) -> None:
     from .mixdb import db_connection
 
-    con = db_connection(location=location, create=True, test=test)
+    con = db_connection(location=location, create=True, test=test, verbose=verbose)
 
     con.execute("""
     CREATE TABLE truth_config(
@@ -130,6 +130,7 @@ def initialize_db(location: str, test: bool = False) -> None:
     snr_gain FLOAT NOT NULL,
     snr_random BOOLEAN NOT NULL,
     start INTEGER NOT NULL,
+    UNIQUE(effects, file_id, pre_tempo, repeat, snr, snr_gain, snr_random, start),
     FOREIGN KEY(file_id) REFERENCES source_file (id))
     """)
 
@@ -155,14 +156,14 @@ def initialize_db(location: str, test: bool = False) -> None:
     con.close()
 
 
-def populate_top_table(location: str, config: dict, test: bool = False) -> None:
+def populate_top_table(location: str, config: dict, test: bool = False, verbose: bool = False) -> None:
     """Populate top table"""
     import json
 
     from .constants import MIXDB_VERSION
     from .mixdb import db_connection
 
-    con = db_connection(location=location, readonly=False, test=test)
+    con = db_connection(location=location, readonly=False, test=test, verbose=verbose)
     con.execute(
         """
     INSERT INTO top (id, asr_configs, class_balancing, feature, mixid_width, num_classes,
@@ -186,11 +187,11 @@ def populate_top_table(location: str, config: dict, test: bool = False) -> None:
     con.close()
 
 
-def populate_class_label_table(location: str, config: dict, test: bool = False) -> None:
+def populate_class_label_table(location: str, config: dict, test: bool = False, verbose: bool = False) -> None:
     """Populate class_label table"""
     from .mixdb import db_connection
 
-    con = db_connection(location=location, readonly=False, test=test)
+    con = db_connection(location=location, readonly=False, test=test, verbose=verbose)
     con.executemany(
         "INSERT INTO class_label (label) VALUES (?)",
         [(item,) for item in config["class_labels"]],
@@ -199,7 +200,12 @@ def populate_class_label_table(location: str, config: dict, test: bool = False)
     con.close()
 
 
-def populate_class_weights_threshold_table(location: str, config: dict, test: bool = False) -> None:
+def populate_class_weights_threshold_table(
+    location: str,
+    config: dict,
+    test: bool = False,
+    verbose: bool = False,
+) -> None:
     """Populate class_weights_threshold table"""
     from .mixdb import db_connection
 
@@ -215,7 +221,7 @@ def populate_class_weights_threshold_table(location: str, config: dict, test: bo
     if len(class_weights_threshold) != num_classes:
         raise ValueError(f"invalid class_weights_threshold length: {len(class_weights_threshold)}")
 
-    con = db_connection(location=location, readonly=False, test=test)
+    con = db_connection(location=location, readonly=False, test=test, verbose=verbose)
     con.executemany(
         "INSERT INTO class_weights_threshold (threshold) VALUES (?)",
         [(item,) for item in class_weights_threshold],
@@ -224,12 +230,12 @@ def populate_class_weights_threshold_table(location: str, config: dict, test: bo
     con.close()
 
 
-def populate_spectral_mask_table(location: str, config: dict, test: bool = False) -> None:
+def populate_spectral_mask_table(location: str, config: dict, test: bool = False, verbose: bool = False) -> None:
     """Populate spectral_mask table"""
     from .config import get_spectral_masks
     from .mixdb import db_connection
 
-    con = db_connection(location=location, readonly=False, test=test)
+    con = db_connection(location=location, readonly=False, test=test, verbose=verbose)
     con.executemany(
         """
     INSERT INTO spectral_mask (f_max_width, f_num, t_max_percent, t_max_width, t_num) VALUES (?, ?, ?, ?, ?)
@@ -249,12 +255,12 @@ def populate_spectral_mask_table(location: str, config: dict, test: bool = False
     con.close()
 
 
-def populate_truth_parameters_table(location: str, config: dict, test: bool = False) -> None:
+def populate_truth_parameters_table(location: str, config: dict, test: bool = False, verbose: bool = False) -> None:
     """Populate truth_parameters table"""
     from .config import get_truth_parameters
     from .mixdb import db_connection
 
-    con = db_connection(location=location, readonly=False, test=test)
+    con = db_connection(location=location, readonly=False, test=test, verbose=verbose)
     con.executemany(
         """
     INSERT INTO truth_parameters (category, name, parameters) VALUES (?, ?, ?)
@@ -272,17 +278,22 @@ def populate_truth_parameters_table(location: str, config: dict, test: bool = Fa
     con.close()
 
 
-def populate_source_file_table(location: str, files: list[SourceFile], test: bool = False) -> None:
+def populate_source_file_table(
+    location: str,
+    files: list[SourceFile],
+    test: bool = False,
+    verbose: bool = False,
+) -> None:
     """Populate source file table"""
     import json
     from pathlib import Path
 
     from .mixdb import db_connection
 
-    _populate_truth_config_table(location, files, test)
-    _populate_speaker_table(location, files, test)
+    _populate_truth_config_table(location, files, test, verbose)
+    _populate_speaker_table(location, files, test, verbose)
 
-    con = db_connection(location=location, readonly=False, test=test)
+    con = db_connection(location=location, readonly=False, test=test, verbose=verbose)
 
     cur = con.cursor()
     textgrid_metadata_tiers: set[str] = set()
@@ -341,13 +352,18 @@ def populate_source_file_table(location: str, files: list[SourceFile], test: boo
     con.close()
 
 
-def populate_impulse_response_file_table(location: str, files: list[ImpulseResponseFile], test: bool = False) -> None:
+def populate_impulse_response_file_table(
+    location: str,
+    files: list[ImpulseResponseFile],
+    test: bool = False,
+    verbose: bool = False,
+) -> None:
     """Populate impulse response file table"""
     from .mixdb import db_connection
 
-    _populate_impulse_response_tag_table(location, files, test)
+    _populate_impulse_response_tag_table(location, files, test, verbose)
 
-    con = db_connection(location=location, readonly=False, test=test)
+    con = db_connection(location=location, readonly=False, test=test, verbose=verbose)
 
     cur = con.cursor()
     for file in files:
@@ -367,12 +383,12 @@ def populate_impulse_response_file_table(location: str, files: list[ImpulseRespo
     con.close()
 
 
-def update_mixid_width(location: str, num_mixtures: int, test: bool = False) -> None:
+def update_mixid_width(location: str, num_mixtures: int, test: bool = False, verbose: bool = False) -> None:
     """Update the mixid width"""
     from ..utils.max_text_width import max_text_width
     from .mixdb import db_connection
 
-    con = db_connection(location=location, readonly=False, test=test)
+    con = db_connection(location=location, readonly=False, test=test, verbose=verbose)
     con.execute("UPDATE top SET mixid_width=? WHERE ? = id", (max_text_width(num_mixtures), 1))
     con.commit()
     con.close()
@@ -422,6 +438,7 @@ def populate_mixture_table(
     location: str,
     mixtures: list[Mixture],
     test: bool = False,
+    verbose: bool = False,
     logging: bool = False,
     show_progress: bool = False,
 ) -> None:
@@ -432,32 +449,11 @@ def populate_mixture_table(
     from .helpers import from_source
     from .mixdb import db_connection
 
-    con = db_connection(location=location, readonly=False, test=test)
+    con = db_connection(location=location, readonly=False, test=test, verbose=verbose)
 
     # Populate source table
     if logging:
-        logger.info("Populating source table")
-    # TODO: refactor this to not load all sources into list; maybe us UNIQUE table modifier?
-    sources: list[tuple[str, int, float, bool, float, float, bool, int]] = []
-    for mixture in mixtures:
-        for source in mixture.all_sources.values():
-            entry = from_source(source)
-            if entry not in sources:
-                sources.append(entry)
-    for source in track(sources, disable=not show_progress):
-        con.execute(
-            """
-        INSERT INTO source (effects, file_id, pre_tempo, repeat, snr, snr_gain, snr_random, start)
-        VALUES (?, ?, ?, ?, ?, ?, ?, ?)
-        """,
-            source,
-        )
-
-    con.commit()
-
-    # Populate mixture table
-    if logging:
-        logger.info("Populating mixture table")
+        logger.info("Populating mixture and source tables")
     for mixture in track(mixtures, disable=not show_progress):
         m_id = int(mixture.name) + 1
         con.execute(
@@ -469,6 +465,14 @@ def populate_mixture_table(
         )
 
         for source in mixture.all_sources.values():
+            con.execute(
+                """
+            INSERT OR IGNORE INTO source (effects, file_id, pre_tempo, repeat, snr, snr_gain, snr_random, start)
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+            """,
+                from_source(source),
+            )
+
             source_id = con.execute(
                 """
                 SELECT id
@@ -487,7 +491,7 @@ def populate_mixture_table(
             con.execute("INSERT INTO mixture_source (mixture_id, source_id) VALUES (?, ?)", (m_id, source_id))
 
     if logging:
-        logger.info("Closing mixture table")
+        logger.info("Closing mixture and source tables")
     con.commit()
     con.close()
 
@@ -857,7 +861,12 @@ def _get_textgrid_tiers_from_source_file(file: str) -> list[str]:
     return sorted(tg.tierNames)
 
 
-def _populate_speaker_table(location: str, source_files: list[SourceFile], test: bool = False) -> None:
+def _populate_speaker_table(
+    location: str,
+    source_files: list[SourceFile],
+    test: bool = False,
+    verbose: bool = False,
+) -> None:
     """Populate speaker table"""
     import json
     from pathlib import Path
@@ -882,7 +891,7 @@ def _populate_speaker_table(location: str, source_files: list[SourceFile], test:
             new_columns.append(column)
     new_columns = sorted(set(new_columns))
 
-    con = db_connection(location=location, readonly=False, test=test)
+    con = db_connection(location=location, readonly=False, test=test, verbose=verbose)
 
     for new_column in new_columns:
         con.execute(f"ALTER TABLE speaker ADD COLUMN {new_column} TEXT")
@@ -914,13 +923,18 @@ def _populate_speaker_table(location: str, source_files: list[SourceFile], test:
     con.close()
 
 
-def _populate_truth_config_table(location: str, source_files: list[SourceFile], test: bool = False) -> None:
+def _populate_truth_config_table(
+    location: str,
+    source_files: list[SourceFile],
+    test: bool = False,
+    verbose: bool = False,
+) -> None:
     """Populate truth_config table"""
     import json
 
     from .mixdb import db_connection
 
-    con = db_connection(location=location, readonly=False, test=test)
+    con = db_connection(location=location, readonly=False, test=test, verbose=verbose)
 
     # Populate truth_config table
     truth_configs: list[str] = []
@@ -938,11 +952,16 @@ def _populate_truth_config_table(location: str, source_files: list[SourceFile],
     con.close()
 
 
-def _populate_impulse_response_tag_table(location: str, files: list[ImpulseResponseFile], test: bool = False) -> None:
+def _populate_impulse_response_tag_table(
+    location: str,
+    files: list[ImpulseResponseFile],
+    test: bool = False,
+    verbose: bool = False,
+) -> None:
     """Populate ir_tag table"""
     from .mixdb import db_connection
 
-    con = db_connection(location=location, readonly=False, test=test)
+    con = db_connection(location=location, readonly=False, test=test, verbose=verbose)
 
     con.executemany(
         "INSERT INTO ir_tag (tag) VALUES (?)",

sonusai/mixture/mixdb.py

@@ -58,6 +58,8 @@ def db_connection(
     from os import remove
     from os.path import exists
 
+    from .. import logger_db
+
     name = db_file(location, test)
     if create and exists(name):
         remove(name)
@@ -71,18 +73,19 @@ def db_connection(
     connection = sqlite3.connect("file:" + name, uri=True, timeout=20)
 
     if verbose:
-        connection.set_trace_callback(print)
+        connection.set_trace_callback(logger_db.debug)
 
     return connection
 
 
 class SQLiteContextManager:
-    def __init__(self, location: str, test: bool = False) -> None:
+    def __init__(self, location: str, test: bool = False, verbose: bool = False) -> None:
         self.location = location
         self.test = test
+        self.verbose = verbose
 
     def __enter__(self) -> Cursor:
-        self.con = db_connection(location=self.location, test=self.test)
+        self.con = db_connection(location=self.location, test=self.test, verbose=self.verbose)
         self.cur = self.con.cursor()
         return self.cur
 
@@ -91,7 +94,7 @@ class SQLiteContextManager:
 
 
 class MixtureDatabase:
-    def __init__(self, location: str, test: bool = False, use_cache: bool = True) -> None:
+    def __init__(self, location: str, test: bool = False, verbose: bool = False, use_cache: bool = True) -> None:
         import json
         from os.path import exists
 
@@ -99,12 +102,13 @@ class MixtureDatabase:
 
         self.location = location
         self.test = test
+        self.verbose = verbose
         self.use_cache = use_cache
 
         if not exists(db_file(self.location, self.test)):
             raise OSError(f"Could not find mixture database in {self.location}")
 
-        self.db = partial(SQLiteContextManager, self.location, self.test)
+        self.db = partial(SQLiteContextManager, self.location, self.test, self.verbose)
 
         # Check config.yml to see if asr_configs has changed and update database if needed
         config = load_config(self.location)
@@ -113,7 +117,7 @@ class MixtureDatabase:
             old_asr_configs = c.execute("SELECT asr_configs FROM top").fetchone()
 
         if old_asr_configs is not None and new_asr_configs != old_asr_configs[0]:
-            con = db_connection(location=self.location, readonly=False, test=self.test)
+            con = db_connection(location=self.location, readonly=False, test=self.test, verbose=self.verbose)
             con.execute("UPDATE top SET asr_configs = ? WHERE ? = id", (new_asr_configs,))
             con.commit()
             con.close()

{sonusai-1.0.4.dist-info → sonusai-1.0.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: sonusai
-Version: 1.0.4
+Version: 1.0.6
 Summary: Framework for building deep neural network models for sound, speech, and voice AI
 Home-page: https://aaware.com
 License: GPL-3.0-only

{sonusai-1.0.4.dist-info → sonusai-1.0.6.dist-info}/RECORD

@@ -1,4 +1,4 @@
-sonusai/__init__.py,sha256=7kXYJuI8EPKezlFD7djImNolS1JMyyltgq_vqlDUCek,3261
+sonusai/__init__.py,sha256=_lBs_XangUYtUcr7JAt55hk1AczFIAgySJfkL-Pt51U,3710
 sonusai/aawscd_probwrite.py,sha256=QZLMQrmPr3OjZ06buyYDwlnk9YPCpyr4KHkBjPsiqjU,3700
 sonusai/audiofe.py,sha256=0DNpntK0WpzNZeyHX8_wC-pDtrgvLkJZFPXz-PspdrY,19448
 sonusai/calc_metric_spenh.py,sha256=0Md6hRFUH9lGnsvoydqne99O7Gi0ieG9vMU_1PnASBg,50019
@@ -23,7 +23,7 @@ sonusai/doc.py,sha256=ZgFSSI56oNDb-yC3xi-RHMClMjryR2VrgGyi3ggX8gM,1098
 sonusai/genft.py,sha256=jGjtjQQEuPunROkoDOYZ7gdyZEa09EMCVhpHrkBZ7A0,5571
 sonusai/genmetrics.py,sha256=sbcqbjI4YOJd5_Lzor4Re_TK6GUQ5zJuYbhDux8odI0,6184
 sonusai/genmix.py,sha256=U62GPgejGfnDfRgXUosmnVVWvTV07sg46JQEIew0nPg,5744
-sonusai/genmixdb.py,sha256=_YPIwBn_yqbE6Nik0j_-3vT0lO35Jemq_xDsA7zoCi4,11132
+sonusai/genmixdb.py,sha256=9T-qn8_Bekc-P7kOY4pV8w-CHixDvSbZPds7wt_mDYU,11272
 sonusai/ir_metric.py,sha256=nxS_mARPSZG5Y0G3L8HysOnkPj4v-RGxAxAVBYe-gJI,19600
 sonusai/lsdb.py,sha256=86t6PpsyardRa6VcSJ-KyU1NiTmlg59VUlcSTptJbn0,5078
 sonusai/main.py,sha256=72feJv5XEVJE_CQatmNIL1VD9ca-Mo0QNDbXxLrHrbQ,2619
@@ -54,12 +54,12 @@ sonusai/mixture/data_io.py,sha256=DV48sFcP2Qp3NBzvcnlptQOXU3aUEcAeLuh3XOtC5jI,53
 sonusai/mixture/db_datatypes.py,sha256=VvNtbOgt5WSeSnBoVcNGC5gs_7hX_38pDUPjy5KRbG4,1471
 sonusai/mixture/effects.py,sha256=ghMO-WiSMQc1CvafD0wkt_DGsM2A6Hi_oZS6j-jeZh8,11784
 sonusai/mixture/feature.py,sha256=7GJvFhfqeqerfjy9Vq9aKt-cecgYblK0IypNNo5hgwY,2285
-sonusai/mixture/generation.py,sha256=iozYcur9oGFDOgovph-F7mzuaD7ie-2ED4NE7kpAPWY,32559
+sonusai/mixture/generation.py,sha256=2VQ41uc1OLFpDKwu0TlcdtxSXwiTJFr_B6_E20pheIY,32844
 sonusai/mixture/helpers.py,sha256=dmyHwf1C5dZjYOd11kVV16KI33CaM-dU_fyaxOrrKt8,11642
 sonusai/mixture/ir_delay.py,sha256=aiC23HMWQ08-v5wORgMx1_DOJSdh4kunULqiQ-SGuMo,2026
 sonusai/mixture/ir_effects.py,sha256=PqiqD4PS42-7kD6ESnsZi2a3tnKCFa4E0xqUujRBvGg,2152
 sonusai/mixture/log_duration_and_sizes.py,sha256=3ekS27IMKlnxIkQAmprzmBnzHOpRjZh3d7maL2VqWQU,927
-sonusai/mixture/mixdb.py,sha256=77GjuedI8a2l6MvQCrz2gDBIZNvwduOQ0ERA-4dXK28,86099
+sonusai/mixture/mixdb.py,sha256=QIpbpS-J0ACNUFoMwZjiylsDgpfkUehLxSq79wcLgzk,86305
 sonusai/mixture/pad_audio.py,sha256=KNxVQAejA0hblLOnMJgLS6lFaeE0n3tWQ5rclaHBnIY,1015
 sonusai/mixture/resample.py,sha256=jXqH6FrZ0mlhQ07XqPx88TT9elu3HHVLw7Q0a7Lh5M4,221
 sonusai/mixture/sox_effects.py,sha256=tndS9qrh3eJOTUPrufyWHCt3UqjbPuh81I4Lo4MNmDg,5328
@@ -132,7 +132,7 @@ sonusai/utils/tokenized_shell_vars.py,sha256=EDrrAgz5lJ0RBAjLcTJt1MeyjhbNZiqXkym
 sonusai/utils/write_audio.py,sha256=IHzrJoFtFcea_J6wo6QSiojRkgnNOzAEcg-z0rFV7nU,810
 sonusai/utils/yes_or_no.py,sha256=0h1okjXmDNbJp7rZJFR2V-HFU1GJDm3YFTUVmYExkOU,263
 sonusai/vars.py,sha256=m8pdgfR4A6A9TCGf_rok6jPAT5BgrEsYXTSISIh1nrI,1163
-sonusai-1.0.4.dist-info/METADATA,sha256=zoOsV6OKVVFihoDmuEyN9mDCLoNPww_3WdB3d-YsJ3Y,2652
-sonusai-1.0.4.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
-sonusai-1.0.4.dist-info/entry_points.txt,sha256=zMNjEphEPO6B3cD1GNpit7z-yA9tUU5-j3W2v-UWstU,92
-sonusai-1.0.4.dist-info/RECORD,,
+sonusai-1.0.6.dist-info/METADATA,sha256=NeaZn2a1yaOmW9AWfWerYv5_RqTTL_MlQGZJv9XHVmo,2652
+sonusai-1.0.6.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
+sonusai-1.0.6.dist-info/entry_points.txt,sha256=zMNjEphEPO6B3cD1GNpit7z-yA9tUU5-j3W2v-UWstU,92
+sonusai-1.0.6.dist-info/RECORD,,