sonusai 0.18.9__py3-none-any.whl → 0.19.5__py3-none-any.whl

sonusai/data/genmixdb.yml

@@ -7,6 +7,8 @@ feature: ""
 
 target_level_type: default
 
+class_indices: 1
+
 targets: [ ]
 
 num_classes: 1
@@ -17,15 +19,7 @@ seed: 0
 
 class_weights_threshold: 0.5
 
-truth_mode: normal
-
-truth_reduction_function: max
-
-truth_settings:
-  function: sed
-  config:
-    thresholds: [ -38, -41, -48 ]
-  index: 1
+truth_configs: { }
 
 asr_manifest: [ ]
 
@@ -52,7 +46,7 @@ snrs:
 
 random_snrs: [ ]
 
-noise_mix_mode: 'exhaustive'
+noise_mix_mode: exhaustive
 
 impulse_responses: [ ]
 
@@ -63,4 +57,4 @@ spectral_masks:
     t_num: 0
     t_max_percent: 100
 
-asr_configs: [ ]
+asr_configs: { }

sonusai/{gentcst.py → deprecated/gentcst.py}

@@ -44,9 +44,9 @@ Outputs:
     gentcst.log
 
 """
+
 import signal
 from dataclasses import dataclass
-from typing import Optional
 
 
 def signal_handler(_sig, _frame):
@@ -54,13 +54,13 @@ def signal_handler(_sig, _frame):
 
     from sonusai import logger
 
-    logger.info('Canceled due to keyboard interrupt')
+    logger.info("Canceled due to keyboard interrupt")
     sys.exit(1)
 
 
 signal.signal(signal.SIGINT, signal_handler)
 
-CONFIG_FILE = 'config.yml'
+CONFIG_FILE = "config.yml"
 
 
 @dataclass
@@ -70,7 +70,7 @@ class FileInfo:
     leaf: str
     labels: list[str]
     fold: int
-    truth_index: Optional[list[int]] = None
+    truth_index: list[int] | None = None
 
 
 @dataclass(frozen=True)
@@ -96,11 +96,13 @@ class OntologyInfo:
     restrictions: list[str]
 
 
-def gentcst(fold: str = '*',
-            ontology: str = 'ontology.json',
-            hierarchical: bool = False,
-            update: bool = False,
-            verbose: bool = False) -> tuple[dict, list[LabelInfo]]:
+def gentcst(
+    fold: str = "*",
+    ontology: str = "ontology.json",
+    hierarchical: bool = False,
+    update: bool = False,
+    verbose: bool = False,
+) -> tuple[dict, list[LabelInfo]]:
     from copy import deepcopy
     from os import getcwd
     from os.path import dirname
@@ -114,25 +116,25 @@ def gentcst(fold: str = '*',
     from sonusai.mixture import get_default_config
     from sonusai.mixture import raw_load_config
     from sonusai.mixture import update_config_from_hierarchy
-    from sonusai.mixture import update_truth_settings
+    from sonusai.mixture import validate_truth_configs
 
     update_console_handler(verbose)
-    initial_log_messages('gentcst')
+    initial_log_messages("gentcst")
 
     if update:
-        logger.info('Updating tree with JSON metadata')
-        logger.info('')
+        logger.info("Updating tree with JSON metadata")
+        logger.info("")
 
-    logger.debug(f'fold:         {fold}')
-    logger.debug(f'ontology:     {ontology}')
-    logger.debug(f'hierarchical: {hierarchical}')
-    logger.debug(f'update:       {update}')
-    logger.debug('')
+    logger.debug(f"fold:         {fold}")
+    logger.debug(f"ontology:     {ontology}")
+    logger.debug(f"hierarchical: {hierarchical}")
+    logger.debug(f"update:       {update}")
+    logger.debug("")
 
     config = get_config()
 
-    if config['truth_mode'] == 'mutex' and hierarchical:
-        raise SonusAIError('Multi-class truth is incompatible with truth_mode mutex')
+    if config["truth_mode"] == "mutex" and hierarchical:
+        raise SonusAIError("Multi-class truth is incompatible with truth_mode mutex")
 
     all_files = get_all_files(hierarchical)
     all_folds = get_folds_from_files(all_files)
@@ -143,52 +145,51 @@ def gentcst(fold: str = '*',
     ontology_data = validate_ontology(ontology, use_files)
 
     labels = get_labels_from_files(all_files)
-    logger.debug('Truth indices:')
+    logger.debug("Truth indices:")
     for item in labels:
-        logger.debug(f' {item.index:3} {item.display_name}')
-    logger.debug('')
+        logger.debug(f" {item.index:3} {item.display_name}")
+    logger.debug("")
 
     gen_truth_indices(use_files, labels)
 
-    config['num_classes'] = len(labels)
-    if config['truth_mode'] == 'mutex':
-        config['num_classes'] = config['num_classes'] + 1
+    config["num_classes"] = len(labels)
+    if config["truth_mode"] == "mutex":
+        config["num_classes"] = config["num_classes"] + 1
 
-    config['targets'] = []
+    config["targets"] = []
 
-    logger.info(f'gentcst {len(use_files)} entries in tree')
+    logger.info(f"gentcst {len(use_files)} entries in tree")
     root = getcwd()
     for file in use_files:
         leaf = dirname(file.file)
         local_config = get_default_config()
         local_config = update_config_from_hierarchy(root=root, leaf=leaf, config=local_config)
-        if not isinstance(local_config['truth_settings'], list):
-            local_config['truth_settings'] = [local_config['truth_settings']]
+        if not isinstance(local_config["truth_settings"], list):
+            local_config["truth_settings"] = [local_config["truth_settings"]]
 
-        specific_name = splitext(file.file)[0] + '.yml'
+        specific_name = splitext(file.file)[0] + ".yml"
         if exists(specific_name):
             specific_config = raw_load_config(specific_name)
 
             for key in specific_config:
-                if key != 'truth_settings':
+                if key != "truth_settings":
                     local_config[key] = specific_config[key]
                 else:
-                    local_config['truth_settings'] = update_truth_settings(given=specific_config['truth_settings'],
-                                                                           default=local_config['truth_settings'])
+                    local_config["truth_settings"] = validate_truth_configs(
+                        given=specific_config["truth_settings"],
+                        default=local_config["truth_settings"],
+                    )
 
-        truth_settings = deepcopy(local_config['truth_settings'])
+        truth_settings = deepcopy(local_config["truth_settings"])
         for idx, val in enumerate(truth_settings):
-            val['index'] = file.truth_index
-            for key in ['function', 'config']:
-                if key in val and val[key] == config['truth_settings'][idx][key]:
+            val["index"] = file.truth_index
+            for key in ["function", "config"]:
+                if key in val and val[key] == config["truth_settings"][idx][key]:
                     del val[key]
 
-        target = {
-            'name':           file.file,
-            'truth_settings': truth_settings
-        }
+        target = {"name": file.file, "truth_settings": truth_settings}
 
-        config['targets'].append(target)
+        config["targets"].append(target)
 
         if update:
             write_metadata_to_tree(ontology_data, file)
@@ -202,28 +203,30 @@ def get_ontology(ontology: str) -> list[OntologyInfo]:
 
     raw_ontology_data = []
     if exists(ontology):
-        with open(file=ontology, mode='r', encoding='utf-8') as f:
+        with open(file=ontology, encoding="utf-8") as f:
             raw_ontology_data = json.load(f)
 
-    return [OntologyInfo(
-        id=item['id'],
-        name=convert_ontology_name(item['name']),
-        description=item['description'],
-        citation_uri=item['citation_uri'],
-        positive_examples=item['positive_examples'],
-        child_ids=item['child_ids'],
-        restrictions=item['restrictions']
-    ) for item in raw_ontology_data]
+    return [
+        OntologyInfo(
+            id=item["id"],
+            name=convert_ontology_name(item["name"]),
+            description=item["description"],
+            citation_uri=item["citation_uri"],
+            positive_examples=item["positive_examples"],
+            child_ids=item["child_ids"],
+            restrictions=item["restrictions"],
+        )
+        for item in raw_ontology_data
+    ]
 
 
 def convert_ontology_name(name: str) -> list[str]:
-    """Convert names to lowercase, convert spaces to '-', split into lists on ','.
-    """
+    """Convert names to lowercase, convert spaces to '-', split into lists on ','."""
     text = name.lower()
     # Need to find ', ' before converting spaces so that we don't convert ', ' to ',-'
-    text = text.replace(', ', ',')
-    text = text.replace(' ', '-')
-    return text.split(',')
+    text = text.replace(", ", ",")
+    text = text.replace(" ", "-")
+    return text.split(",")
 
 
 def get_dirs() -> list[DirInfo]:
@@ -235,13 +238,13 @@ def get_dirs() -> list[DirInfo]:
     cwd = getcwd()
     for root, ds, _ in walk(cwd):
         for d in ds:
-            match.append(join(root, d).replace(cwd, '.'))
+            match.append(join(root, d).replace(cwd, "."))
     match = sorted(match)
 
     dirs: list[DirInfo] = []
     for m in match:
-        if not m.startswith('./gentcst') and m != '.':
-            classes = m.split('/')
+        if not m.startswith("./gentcst") and m != ".":
+            classes = m.split("/")
             # Remove first element because it is '.'
             classes.pop(0)
             dirs.append(DirInfo(file=m, classes=classes))
@@ -250,7 +253,7 @@ def get_dirs() -> list[DirInfo]:
 
 
 def get_leaf_from_classes(classes: list[str]) -> str:
-    return '/'.join(classes)
+    return "/".join(classes)
 
 
 def get_all_files(hierarchical: bool) -> list[FileInfo]:
@@ -261,20 +264,20 @@ def get_all_files(hierarchical: bool) -> list[FileInfo]:
 
     file_list: list[str] = []
     cwd = getcwd()
-    regex = re.compile(r'^\d+-.*\.txt$')
+    regex = re.compile(r"^\d+-.*\.txt$")
     for root, _, fs in walk(cwd):
         for f in fs:
             if regex.match(f):
-                file_list.append(join(root, f).replace(cwd, '.'))
+                file_list.append(join(root, f).replace(cwd, "."))
 
     files: list[FileInfo] = []
-    fold_pattern = re.compile(r'.*/(\d+)-.*\.txt')
+    fold_pattern = re.compile(r".*/(\d+)-.*\.txt")
     for file in file_list:
         fold_match = fold_pattern.match(file)
         if fold_match:
             fold = int(fold_match.group(1))
 
-            classes = file.split('/')
+            classes = file.split("/")
             # Remove first element because it is '.'
             classes.pop(0)
             # Remove last element because it is the file name
@@ -287,11 +290,7 @@ def get_all_files(hierarchical: bool) -> list[FileInfo]:
             else:
                 labels = [leaf]
 
-            files.append(FileInfo(file=file,
-                                  classes=classes,
-                                  leaf=leaf,
-                                  labels=labels,
-                                  fold=fold))
+            files.append(FileInfo(file=file, classes=classes, leaf=leaf, labels=labels, fold=fold))
 
     return files
 
@@ -304,13 +303,13 @@ def get_use_folds(all_folds: list[int], fold: str) -> list[int]:
     req_folds: list[int] = get_req_folds(all_folds, fold)
     use_folds: list[int] = list(set(req_folds).intersection(all_folds))
 
-    logger.debug('Fold information')
-    logger.debug(f' Available: {all_folds}')
-    logger.debug(f' Requested: {req_folds}')
-    logger.debug(f' Used:      {use_folds}')
-    logger.debug(f' Unused:    {list(set(use_folds).symmetric_difference(all_folds))}')
-    logger.debug(f' Missing:   {list(np.setdiff1d(req_folds, all_folds))}')
-    logger.debug('')
+    logger.debug("Fold information")
+    logger.debug(f" Available: {all_folds}")
+    logger.debug(f" Requested: {req_folds}")
+    logger.debug(f" Used:      {use_folds}")
+    logger.debug(f" Unused:    {list(set(use_folds).symmetric_difference(all_folds))}")
+    logger.debug(f" Missing:   {list(np.setdiff1d(req_folds, all_folds))}")
+    logger.debug("")
 
     return use_folds
 
@@ -319,15 +318,15 @@ def get_files_from_folds(files: list[FileInfo], folds: list[int]) -> list[FileIn
     return [file for file in files if file.fold in folds]
 
 
-def get_item_from_name(ontology_data: list[OntologyInfo], name: str) -> OntologyInfo:
+def get_item_from_name(ontology_data: list[OntologyInfo], name: str) -> OntologyInfo | None:
     return next((item for item in ontology_data if name in item.name), None)
 
 
-def get_item_from_id(ontology_data: list[OntologyInfo], identity: str) -> OntologyInfo:
+def get_item_from_id(ontology_data: list[OntologyInfo], identity: str) -> OntologyInfo | None:
     return next((item for item in ontology_data if item.id == identity), None)
 
 
-def get_name_from_id(ontology_data: list[OntologyInfo], identity: str) -> list[str]:
+def get_name_from_id(ontology_data: list[OntologyInfo], identity: str) -> list[str] | None:
     from sonusai import SonusAIError
 
     name = None
@@ -336,7 +335,7 @@ def get_name_from_id(ontology_data: list[OntologyInfo], identity: str) -> list[s
     for item in ontology_data:
         if item.id == identity:
             if found:
-                raise SonusAIError(f'id {identity} appears multiple times in ontology')
+                raise SonusAIError(f"id {identity} appears multiple times in ontology")
 
             name = item.name
             found = True
@@ -344,7 +343,7 @@ def get_name_from_id(ontology_data: list[OntologyInfo], identity: str) -> list[s
     return name
 
 
-def get_id_from_name(ontology_data: list[OntologyInfo], name: str) -> str:
+def get_id_from_name(ontology_data: list[OntologyInfo], name: str) -> str | None:
     from sonusai import SonusAIError
 
     identity = None
@@ -353,7 +352,7 @@ def get_id_from_name(ontology_data: list[OntologyInfo], name: str) -> str:
     for item in ontology_data:
         if name in item.name:
             if found:
-                raise SonusAIError(f'name {name} appears multiple times in ontology')
+                raise SonusAIError(f"name {name} appears multiple times in ontology")
 
             identity = item.id
             found = True
@@ -362,10 +361,7 @@ def get_id_from_name(ontology_data: list[OntologyInfo], name: str) -> str:
 
 
 def is_valid_name(ontology_data: list[OntologyInfo], name: str) -> bool:
-    for item in ontology_data:
-        if name in item.name:
-            return True
-    return False
+    return any(name in item.name for item in ontology_data)
 
 
 def is_valid_child(ontology_data: list[OntologyInfo], parent: str, child: str) -> bool:
@@ -373,17 +369,17 @@ def is_valid_child(ontology_data: list[OntologyInfo], parent: str, child: str) -
     parent_item = get_item_from_name(ontology_data, parent)
     child_id = get_id_from_name(ontology_data, child)
 
-    if child_id is not None and parent_item is not None:
-        if child_id in parent_item.child_ids:
-            valid = True
+    if child_id is not None and parent_item is not None and child_id in parent_item.child_ids:
+        valid = True
 
     return valid
 
 
 def is_valid_hierarchy(ontology_data: list[OntologyInfo], classes: list[str]) -> bool:
+    from itertools import pairwise
     valid = True
 
-    for parent, child in zip(classes, classes[1:]):
+    for parent, child in pairwise(classes):
         if not is_valid_child(ontology_data, parent, child):
             valid = False
 
@@ -397,13 +393,12 @@ def validate_class(ontology_data: list[OntologyInfo], item: FileInfo | DirInfo)
 
     for c in item.classes:
         if not is_valid_name(ontology_data, c):
-            logger.warning(f' Could not find {c} in ontology for {item.file}')
+            logger.warning(f" Could not find {c} in ontology for {item.file}")
             valid = False
 
-    if valid:
-        if not is_valid_hierarchy(ontology_data, item.classes):
-            logger.warning(f' Invalid parent/child relationship for {item.file}')
-            valid = False
+    if valid and not is_valid_hierarchy(ontology_data, item.classes):
+        logger.warning(f" Invalid parent/child relationship for {item.file}")
+        valid = False
 
     return valid
 
@@ -411,7 +406,7 @@ def validate_class(ontology_data: list[OntologyInfo], item: FileInfo | DirInfo)
 def get_req_folds(folds: list[int], fold: str) -> list[int]:
     from sonusai.utils import expand_range
 
-    if fold == '*':
+    if fold == "*":
         return folds
 
     return expand_range(fold)
@@ -420,13 +415,13 @@ def get_req_folds(folds: list[int], fold: str) -> list[int]:
 def get_folds_from_files(files: list[FileInfo]) -> list[int]:
     result: list[int] = [file.fold for file in files]
     # Converting to set and back to list ensures uniqueness
-    return sorted(list(set(result)))
+    return sorted(set(result))
 
 
 def get_leaves_from_files(files: list[FileInfo]) -> list[str]:
     result: list[str] = [file.leaf for file in files]
     # Converting to set and back to list ensures uniqueness
-    return sorted(list(set(result)))
+    return sorted(set(result))
 
 
 def get_labels_from_files(files: list[FileInfo]) -> list[LabelInfo]:
@@ -440,7 +435,7 @@ def get_labels_from_files(files: list[FileInfo]) -> list[LabelInfo]:
 
     for n in range(len(labels_by_depth)):
         # Converting to set and back to list ensures uniqueness
-        labels_by_depth[n] = sorted(list(set(labels_by_depth[n])))
+        labels_by_depth[n] = sorted(set(labels_by_depth[n]))
 
     # We want the deepest leaves first
     labels_by_depth.reverse()
@@ -471,21 +466,21 @@ def get_config() -> dict:
     from sonusai.mixture import raw_load_config
 
     if not exists(CONFIG_FILE):
-        raise SonusAIError(f'No {CONFIG_FILE} at top level')
+        raise SonusAIError(f"No {CONFIG_FILE} at top level")
 
     config = raw_load_config(CONFIG_FILE)
 
-    if 'feature' not in config:
-        raise SonusAIError('feature not in top level config')
+    if "feature" not in config:
+        raise SonusAIError("feature not in top level config")
 
-    if 'truth_mode' not in config:
-        raise SonusAIError('truth_mode not in top level config')
+    if "truth_mode" not in config:
+        raise SonusAIError("truth_mode not in top level config")
 
-    if config['truth_mode'] not in ['normal', 'mutex']:
-        raise SonusAIError('Invalid truth_mode in top level config')
+    if config["truth_mode"] not in ["normal", "mutex"]:
+        raise SonusAIError("Invalid truth_mode in top level config")
 
-    if 'truth_settings' in config and not isinstance(config['truth_settings'], list):
-        config['truth_settings'] = [config['truth_settings']]
+    if "truth_settings" in config and not isinstance(config["truth_settings"], list):
+        config["truth_settings"] = [config["truth_settings"]]
 
     return config
 
@@ -497,34 +492,34 @@ def validate_ontology(ontology: str, items: list[FileInfo] | list[DirInfo]) -> l
 
     if exists(ontology):
         ontology_data = get_ontology(ontology)
-        logger.debug(f'Reference ontology in {ontology} has {len(ontology_data)} classes')
-        logger.debug('')
+        logger.debug(f"Reference ontology in {ontology} has {len(ontology_data)} classes")
+        logger.debug("")
 
-        logger.info('Checking tree against reference ontology')
+        logger.info("Checking tree against reference ontology")
         all_dirs = get_dirs()
         valid = True
         for file in all_dirs:
             if not validate_class(ontology_data, file):
                 valid = False
         if valid:
-            logger.info('PASS')
-        logger.info('')
+            logger.info("PASS")
+        logger.info("")
 
-        logger.info('Checking files against reference ontology')
+        logger.info("Checking files against reference ontology")
         valid = True
         for item in items:
             if not validate_class(ontology_data, item):
                 valid = False
         if valid:
-            logger.info('PASS')
-        logger.info('')
+            logger.info("PASS")
+        logger.info("")
 
         return ontology_data
 
     return []
 
 
-def get_node_from_name(ontology_data: list[OntologyInfo], name: str) -> Optional[OntologyInfo]:
+def get_node_from_name(ontology_data: list[OntologyInfo], name: str) -> OntologyInfo | None:
     from sonusai import logger
 
     nodes = [item for item in ontology_data if name in item.name]
@@ -532,9 +527,9 @@ def get_node_from_name(ontology_data: list[OntologyInfo], name: str) -> Optional
         return nodes[0]
 
     if nodes:
-        logger.warning(f'Found multiple entries in reference ontology that match {name}')
+        logger.warning(f"Found multiple entries in reference ontology that match {name}")
     else:
-        logger.warning(f'Could not find entry for {name} in reference ontology')
+        logger.warning(f"Could not find entry for {name} in reference ontology")
 
     return None
 
@@ -549,8 +544,8 @@ def write_metadata_to_tree(ontology_data: list[OntologyInfo], file: FileInfo):
         node = get_node_from_name(ontology_data, file.classes[-1])
         if node is not None:
             dir_name = dirname(file.file)
-            json_name = dir_name + '/' + basename(dir_name) + '.json'
-            with open(file=json_name, mode='w') as f:
+            json_name = dir_name + "/" + basename(dir_name) + ".json"
+            with open(file=json_name, mode="w") as f:
                 json.dump(asdict(node), f)
 
 
@@ -565,19 +560,19 @@ def report_leaf_fold_data_usage(all_files: list[FileInfo], use_files: list[FileI
     use_leaves = get_leaves_from_files(use_files)
     all_leaves = get_leaves_from_files(all_files)
 
-    logger.debug('Data folds present in each leaf')
+    logger.debug("Data folds present in each leaf")
     leaf_len = len(max(all_leaves, key=len))
     for leaf in all_leaves:
         folds = get_folds_from_leaf(all_files, leaf)
-        logger.debug(f' {leaf:{leaf_len}} {folds}')
-    logger.debug('')
+        logger.debug(f" {leaf:{leaf_len}} {folds}")
+    logger.debug("")
 
     dif_leaves = set(all_leaves).symmetric_difference(use_leaves)
     if dif_leaves:
-        logger.warning('This fold selection is missing data from the following leaves')
+        logger.warning("This fold selection is missing data from the following leaves")
         for c in dif_leaves:
-            logger.warning(f' {c}')
-        logger.warning('')
+            logger.warning(f" {c}")
+        logger.warning("")
 
 
 def main() -> None:
@@ -588,12 +583,12 @@ def main() -> None:
 
     args = docopt(trim_docstring(__doc__), version=sonusai.__version__, options_first=True)
 
-    verbose = args['--verbose']
-    output_name = args['--output']
-    fold = args['--fold']
-    ontology = args['--ontology']
-    hierarchical = args['--hierarchical']
-    update = args['--update']
+    verbose = args["--verbose"]
+    output_name = args["--output"]
+    fold = args["--fold"]
+    ontology = args["--ontology"]
+    hierarchical = args["--hierarchical"]
+    update = args["--update"]
 
     import csv
     from dataclasses import asdict
@@ -607,29 +602,31 @@ def main() -> None:
     from sonusai import logger
 
     if not output_name:
-        output_name = basename(getcwd()) + '.yml'
+        output_name = basename(getcwd()) + ".yml"
 
-    create_file_handler('gentcst.log')
+    create_file_handler("gentcst.log")
 
-    config, labels = gentcst(fold=fold,
-                             ontology=ontology,
-                             hierarchical=hierarchical,
-                             update=update,
-                             verbose=verbose)
+    config, labels = gentcst(
+        fold=fold,
+        ontology=ontology,
+        hierarchical=hierarchical,
+        update=update,
+        verbose=verbose,
+    )
 
-    with open(file=output_name, mode='w') as f:
+    with open(file=output_name, mode="w") as f:
         yaml.dump(config, f)
-        logger.info(f'Wrote config to {output_name}')
+        logger.info(f"Wrote config to {output_name}")
 
-    csv_fields = ['index', 'display_name']
-    csv_name = splitext(output_name)[0] + '.csv'
+    csv_fields = ["index", "display_name"]
+    csv_name = splitext(output_name)[0] + ".csv"
 
-    with open(file=csv_name, mode='w') as f:
+    with open(file=csv_name, mode="w") as f:
         writer = csv.DictWriter(f, fieldnames=csv_fields)
         writer.writeheader()
         writer.writerows([asdict(label) for label in labels])
-        logger.info(f'Wrote labels to {csv_name}')
+        logger.info(f"Wrote labels to {csv_name}")
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()