sonusai 0.17.3__py3-none-any.whl → 0.18.0__py3-none-any.whl

{sonusai-0.17.3.dist-info → sonusai-0.18.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sonusai
-Version: 0.17.3
+Version: 0.18.0
 Summary: Framework for building deep neural network models for sound, speech, and voice AI
 Home-page: https://aaware.com
 License: GPL-3.0-only
@@ -21,6 +21,7 @@ Requires-Dist: h5py (>=3.11.0,<4.0.0)
 Requires-Dist: jiwer (>=3.0.3,<4.0.0)
 Requires-Dist: librosa (>=0.10.1,<0.11.0)
 Requires-Dist: matplotlib (>=3.8.0,<4.0.0)
+Requires-Dist: mgzip (>=0.2.1,<0.3.0)
 Requires-Dist: numpy (>=1.26.4,<2.0.0)
 Requires-Dist: onnx (>=1.14.1,<2.0.0)
 Requires-Dist: onnxruntime (>=1.16.1,<2.0.0)
@@ -28,6 +29,7 @@ Requires-Dist: paho-mqtt (>=2.0.0,<3.0.0)
 Requires-Dist: pandas (>=2.1.1,<3.0.0)
 Requires-Dist: pesq (>=0.0.4,<0.0.5)
 Requires-Dist: praatio (>=6.2.0,<7.0.0)
+Requires-Dist: psutil (>=5,<6)
 Requires-Dist: pyaaware (>=1.5.7,<2.0.0)
 Requires-Dist: pyaudio (>=0.2.14,<0.3.0)
 Requires-Dist: pydub (>=0.25.1,<0.26.0)

{sonusai-0.17.3.dist-info → sonusai-0.18.0.dist-info}/RECORD

@@ -1,17 +1,18 @@
-sonusai/__init__.py,sha256=vzTFfRB-NeO-Sm3puySDJOybk3ND_Oj6w0EejQPmH1U,2978
+sonusai/__init__.py,sha256=j2eH_QUsIIMm0HDiNHC5HCocWsX_GhtvlmTkT7zyYOw,2918
 sonusai/aawscd_probwrite.py,sha256=GukR5owp_0A3DrqSl9fHWULYgclNft4D5OkHIwfxxkc,3698
 sonusai/audiofe.py,sha256=zOySiYs5ZZm60eMbA7RjhG6C0Ouhaii3WfL1d0Q8rxg,11154
-sonusai/calc_metric_spenh.py,sha256=0UvJibVjjoHTty3QFh0qzugXe-LvIj2XsMwoaIR44BM,62171
+sonusai/calc_metric_spenh.py,sha256=SunJD8wkdUxyL0rRZt2auauZBEUzpi0IRY8MtXKh3wo,63645
 sonusai/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sonusai/data/genmixdb.yml,sha256=-XSs_hUR6wHJVoTPmSewzXL7u61X-xmHY46lNPatxSE,1025
 sonusai/data/speech_ma01_01.wav,sha256=PK0vMKg-NR6rPE3KouxHGF6PKXnJCr7AwjMqfu98LUA,76644
 sonusai/data/whitenoise.wav,sha256=I2umov0m34y56F9IsIBi1XtE76ZeZaSKDf70cJRe3pI,1920044
 sonusai/doc/__init__.py,sha256=rP5Hgn0Iys_xkuv4caxngdqehuU4zLZsiKuv8Nde67M,19
-sonusai/doc/doc.py,sha256=4NEZ2K-hTk7Y1Gxx09UEjNhiYkD9xid-kJ1Nt8H5_gM,22670
+sonusai/doc/doc.py,sha256=LOf4HiIxMeQ7-n0ExDSldo1Tt1036SVaN_auwlqXUxQ,21489
 sonusai/doc.py,sha256=l8CaFgLI8mqx4tn0aXfxKqa2dy9GgC0zjYxZAkpmi1E,878
 sonusai/genft.py,sha256=OzET3iTE-QhrUckzidfZvCDXZlAxIF5Xe5NEf856Vvk,5662
+sonusai/genmetrics.py,sha256=fC8KPIB6wtBj_hs1X65lq3dqNTcWeuNs1eT7yXdpxD8,1830
 sonusai/genmix.py,sha256=TU5aTebGHsbfwsRbynYbegGBelSma9khuQkDk0dFE3I,7075
-sonusai/genmixdb.py,sha256=M67Y_SEysgHfTmHHOdOjxdpuryTMDNgbDteCzR1uLk8,19669
+sonusai/genmixdb.py,sha256=gF2qas1tH9MHEFLoEMrN3kYVm-vhAKaOuZ8ev-w4MQM,19553
 sonusai/gentcst.py,sha256=W1ZO3xs7CoZkFcvOTH-FLJOIA4I7Wzb0HVRC3hGGSaM,20223
 sonusai/lsdb.py,sha256=fMRqPlAu4B-4MsTXX-NaWXYyJ_dAOJlS-LrvQPQQsXg,6028
 sonusai/main.py,sha256=GC-pQrSqx9tWwIcmEo6V9SraEv5KskBLS_W_wz-f2ZM,2509
@@ -28,28 +29,27 @@ sonusai/metrics/class_summary.py,sha256=4Mb25nuk6eqotnQSFMuOQL3zofGcpNXDfDlPa513
 sonusai/metrics/confusion_matrix_summary.py,sha256=3qg6TMKjJeHtNjj2YnNjPFSlMrQXt0Zcu1dLkGB_aPU,4001
 sonusai/metrics/one_hot.py,sha256=QSeH_GdqBpOAKLrNnQ8gjcPC-vSdUqC0yPEQueTA6VI,13548
 sonusai/metrics/snr_summary.py,sha256=P4U5_Xr7v9F8kF-rZBnpsVNt3p42rIVS6zmch8yfVfg,5575
-sonusai/mixture/__init__.py,sha256=yszEbRnlxeZXSegEBUVwyrSZwNIl6ufaJu_NiZ-1rqY,5399
+sonusai/mixture/__init__.py,sha256=rFaHyroCf0Fd-SuxmH4dl8xZVjOe8gFjndouv9RtzIE,5347
 sonusai/mixture/audio.py,sha256=2lqy0DtTMTYhX4aAOIvVtLNn6QB5ivTa7cJIaAlbfAg,2385
 sonusai/mixture/augmentation.py,sha256=Blb90tdTwBOj5w9tRcYyS5H67YJuFiXsGqwZWd7ON4g,10468
 sonusai/mixture/class_count.py,sha256=_wFnVl2yEOnbor7pLg7cYOUeX6nioov-03Cv3SEbh2k,996
 sonusai/mixture/config.py,sha256=d2IzZ1samHWGMpoKzSmUwMyAWWhgmyNoxyO8oiUwbsg,22193
-sonusai/mixture/constants.py,sha256=xjCskcQi6khqYZDf7j6z1OkeN1C6wE06kBBapcJiNI4,1428
-sonusai/mixture/datatypes.py,sha256=mMNxtzyDvAmtuoTHVVJP7jBi6OH-QyC1NfC_ZIiuLlY,8440
+sonusai/mixture/constants.py,sha256=ZRM7Z8e6EwnL9RwaMVxks-QokN9KMWxnQzAf9VNxa9M,1408
+sonusai/mixture/datatypes.py,sha256=uVJtT2sVGS531pSglhaLLF5hZcI3_0oKQOWmMkrCwqo,8253
 sonusai/mixture/eq_rule_is_valid.py,sha256=MpQwRA5M76wSiQWEI1lW2cLFdPaMttBLcQp3tWD8efM,1243
 sonusai/mixture/feature.py,sha256=Rwuf82IoXzhHPGbKYVGcatImF_ssBf_FfvbqghVPXtg,4116
-sonusai/mixture/generation.py,sha256=miUrc3QOSUNIG6mDkiMCZ6M2ulivUZxlYUAJUOVomWc,39039
-sonusai/mixture/helpers.py,sha256=GSGSD2KnvOeEIB6IwNTxyaQNjghTSBMB729kUEd_RiM,22403
+sonusai/mixture/generation.py,sha256=H_9kgresvNAEI6pmqAEwOaxQMPg6geVw6G0Y8fmbu_o,42580
+sonusai/mixture/helpers.py,sha256=4gb_t65Aw9O8hUgDzq-7D_t6LynfQDl1jcuFhQIAdWI,24590
 sonusai/mixture/log_duration_and_sizes.py,sha256=baTUpqyM15wA125jo9E3posmVJUe3WlpksyO6v9Jul0,1347
-sonusai/mixture/mapped_snr_f.py,sha256=mlbYM1t14OXe_Zg4CjpWTuA_Zun4W0O3bSUXeodRBQs,1845
-sonusai/mixture/mixdb.py,sha256=zfBaVOuMKHTDzmwR4hWRpGxFwnV_fJkwLDmn9M1SDds,49791
+sonusai/mixture/mapped_snr_f.py,sha256=Fdf2uw62FvyKvVy5VywaUtPZGO1zCWQsHlte0bwkKPQ,3121
+sonusai/mixture/mixdb.py,sha256=xgHD43OgXaFKb1o4nsCPt-bd_mnk-PPetX4OGCGZ8DM,51582
 sonusai/mixture/soundfile_audio.py,sha256=mHa5SIXsu_uE0j3DO52GydRJrvWSzU_nII-7YJfQ6Qo,4154
 sonusai/mixture/sox_audio.py,sha256=HT3kYA9TP5QPCuoOJdUMnGVN-qY6q96DGL8zxuog76o,12277
 sonusai/mixture/sox_augmentation.py,sha256=kBWPrsFk0EBi71nLcKt5v0GA34bY7g9D9x0cEamNWbU,4564
-sonusai/mixture/speaker_metadata.py,sha256=l98avdxLYUsSDZ88xUjfvHnACkbnD0_Dg1aBGDbzS9I,1380
 sonusai/mixture/spectral_mask.py,sha256=8AkCwhy-PSdP1Uri9miKZP-bXFYnFcH_c9xZCGrHavU,2071
 sonusai/mixture/target_class_balancing.py,sha256=NTNiKZH0_PWLooeow0l41CjJKK8ZTMVbUqz9ZkaNtWk,4900
 sonusai/mixture/targets.py,sha256=wyy5vhLhuN-hqBMBGoziVvEJg3FKFvJFgmEE7_LaV2M,7908
-sonusai/mixture/tokenized_shell_vars.py,sha256=gCxw8SQUcal6mqWKF7hOBTgSQmbJUk1nT0Gn3H8GA0U,4705
+sonusai/mixture/tokenized_shell_vars.py,sha256=zIAFvwP2WSvkMAGY7f3SJ4KLXI6IBT-U_e9ptnoo5Hc,4803
 sonusai/mixture/torchaudio_audio.py,sha256=KhHeOMsjmbwOaAcoKD61aFvYBYSlA8OysfT5iGn45MA,3010
 sonusai/mixture/torchaudio_augmentation.py,sha256=1vEDHI0caL1vrgoY2lAWe4CiHE2jKRuKKH7x23GHw0w,4390
 sonusai/mixture/truth.py,sha256=Y41pZ52Xkols9LUler0NlgnilUOscBIucmw4GcxXNzU,1612
@@ -61,7 +61,6 @@ sonusai/mixture/truth_functions/file.py,sha256=jOJuC_3y9BH6GGOp9eKcbVrHLVRzUA80B
 sonusai/mixture/truth_functions/phoneme.py,sha256=stYdlPuNytQK_LLT61OJLfYSqKd-sDjQZdtJKGzt5wA,479
 sonusai/mixture/truth_functions/sed.py,sha256=8cHjEFjZaH_0hIOHhPmj4AJz2GpEADM6Ys2x4NoiWSY,2469
 sonusai/mixture/truth_functions/target.py,sha256=KAsjugDRooOA5BRcHVAbZRgV7l8S5CFg7CZ0XtKZaQ0,5764
-sonusai/mkmanifest.py,sha256=imI8swwPYVzumrUYEL-9JLvun-ez98PtlUBj2b729k8,8682
 sonusai/mkwav.py,sha256=zfSyIiQTIK3KV9Ij33jkLhhZIMVYqaROcRQ4S7c4sIo,5364
 sonusai/onnx_predict.py,sha256=jSxhD2oFyGSTHOGCXbW4fRT-k4SqKOboK2JaDO-yWcs,8737
 sonusai/plot.py,sha256=ERkmxMM3qjcCDm4LGDQY4fRAncCYAzP7uW8iZ7_brcg,17105
@@ -72,7 +71,7 @@ sonusai/speech/__init__.py,sha256=SuPcU_K9wQISsZRIzsRNLtEC6cb616l-Jlx3PU-HWMs,11
 sonusai/speech/l2arctic.py,sha256=28TT3CohvPu98YNUb8O7rWHAYgPGwYTOLSdfNQjOuyc,3736
 sonusai/speech/librispeech.py,sha256=A0IpamojCPXyJiHcjCtI7yNWdMjB00cbggjHslssrg8,3120
 sonusai/speech/mcgill.py,sha256=jcddj64fLdV3sO6CJNafm3w-2SnYoyQtU90odXhdaaE,1976
-sonusai/speech/textgrid.py,sha256=8hB6SdEEXxo6JXVFq8mJ1-ilRbBiRXhaHTQjA-HWg-0,3385
+sonusai/speech/textgrid.py,sha256=MVgpZhoL_ZXffqzqgC4N_EVmVpYP3FaOaNP9iTDUsUU,2722
 sonusai/speech/timit.py,sha256=1vWgj6isD3ATOjMJSTjOPLmDkYyB65M5MwYipEmLEvg,4081
 sonusai/speech/types.py,sha256=4eKVPAktpkIrZ2qoVp2iT45zxTVNocQEGT6O_Zlub_w,214
 sonusai/speech/vctk.py,sha256=EAMEBAzjZUI6dw15n-yI2oCN-H4tzM9t4aUVlOxpAbo,1540
@@ -84,11 +83,6 @@ sonusai/utils/asl_p56.py,sha256=-bvQpd-jRQVURbkZJpRoyEAq6gTv9Rc3oFDbh5_lcjY,3861
 sonusai/utils/asr.py,sha256=6y6VYJizHpuQ3MgKbEQ4t2gofO-MW6Ez23oAd6d23IE,2920
 sonusai/utils/asr_functions/__init__.py,sha256=JyHK67s97bw7QzrlkboWhws4yNytdPatqzLJxfwx-yw,43
 sonusai/utils/asr_functions/aaware_whisper.py,sha256=Ew3zb8OfbxEW7q1s-KA7D5eph4SjVSUAJgiLK-vVqhI,1985
-sonusai/utils/asr_manifest_functions/__init__.py,sha256=jfi9xC5c86F_aMSsI5Xj-pxWGxuQ7fwZ8Wdf4T7kDsA,343
-sonusai/utils/asr_manifest_functions/data.py,sha256=nO4oT3EQmydwn1pzc-ZM09yz4X2ic-LQuHzGEnJhKe8,32
-sonusai/utils/asr_manifest_functions/librispeech.py,sha256=_3tGc8qfAUpYJZ0_avpW0vGp7zjdpeqj1HAgXi3TL4Q,1612
-sonusai/utils/asr_manifest_functions/mcgill_speech.py,sha256=dW-5XTC5xOY3PHU2DvlWNWDeoprXDD0Zq2dXDdPAjzE,934
-sonusai/utils/asr_manifest_functions/vctk_noisy_speech.py,sha256=9iMrnE-qabLMnyewyxsBMl0uCS8yS7BPJOdmUoOnGAc,2146
 sonusai/utils/audio_devices.py,sha256=LgaXTln1oRArBzaet3rZiIO2plgtaThuGBc3sJ_sLlo,1414
 sonusai/utils/braced_glob.py,sha256=Z_XIpPK17QiP1JbzAnUC5w3oyG8ZovoyM22Wh-Q_vWU,1675
 sonusai/utils/calculate_input_shape.py,sha256=63ILxibYKuTQozY83QN8Y2OOhBEbW_1X47Q0askcHDM,984
@@ -121,7 +115,7 @@ sonusai/utils/stratified_shuffle_split.py,sha256=rJNXvBp-GxoKzH3OpL7k0ANSu5xMP2z
 sonusai/utils/write_audio.py,sha256=ZsPGExwM86QHLLN2LOWekK2uAqf5pV_1oRW811p0QAI,840
 sonusai/utils/yes_or_no.py,sha256=eMLXBVH0cEahiXY4W2KNORmwNQ-ba10eRtldh0y4NYg,263
 sonusai/vars.py,sha256=m2AefF0m5bXWGXpJj8Pi42zWL2ydeEj7bkak3GrtMyM,940
-sonusai-0.17.3.dist-info/METADATA,sha256=Zu3DK9fCT1xiipWOBY4JhUZJ3qm1cVRc5eLAXonDb2s,2522
-sonusai-0.17.3.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-sonusai-0.17.3.dist-info/entry_points.txt,sha256=zMNjEphEPO6B3cD1GNpit7z-yA9tUU5-j3W2v-UWstU,92
-sonusai-0.17.3.dist-info/RECORD,,
+sonusai-0.18.0.dist-info/METADATA,sha256=LwOGcp1V_87ef8oyjI8Kjwo5K8tbIKjpRfJ17dW0Dbc,2591
+sonusai-0.18.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+sonusai-0.18.0.dist-info/entry_points.txt,sha256=zMNjEphEPO6B3cD1GNpit7z-yA9tUU5-j3W2v-UWstU,92
+sonusai-0.18.0.dist-info/RECORD,,

sonusai/mixture/speaker_metadata.py

@@ -1,35 +0,0 @@
-from functools import cached_property
-from pathlib import Path
-
-from praatio import textgrid
-from praatio.data_classes.textgrid_tier import TextgridTier
-from praatio.utilities.constants import Interval
-
-from sonusai.mixture.datatypes import TargetFiles
-from sonusai.mixture.tokenized_shell_vars import tokenized_expand
-
-
-class SpeakerMetadata:
-    def __init__(self, target_files: TargetFiles) -> None:
-        self.data: dict[str, dict[str, TextgridTier]] = {}
-        for file in target_files:
-            self.data[file.name] = {}
-            file_name, _ = tokenized_expand(file.name)
-            tg_file = Path(file_name).with_suffix('.TextGrid')
-            if tg_file.exists():
-                tg = textgrid.openTextgrid(str(tg_file), includeEmptyIntervals=False)
-                for tier in tg.tierNames:
-                    self.data[file.name][tier] = tg.getTier(tier)
-
-    @cached_property
-    def tiers(self) -> list[str]:
-        return sorted(list(set([key for value in self.data.values() for key in value.keys()])))
-
-    def all(self, tier: str, label_only: bool = False) -> list[Interval]:
-        results = [value[tier].entries for value in self.data.values()]
-        if label_only:
-            return sorted(set([r.label for result in results for r in result]))
-        return results
-
-    def mixids_for(self, tier: str, value: str) -> list[int]:
-        pass

sonusai/mkmanifest.py

@@ -1,209 +0,0 @@
-"""mkmanifest
-
-usage: mkmanifest [-hvn] [--include GLOB] [-m METHOD] [-e ADAT] [-o OUTPUT] PATH ...
-
-options:
-    -h, --help
-    -v, --verbose                   Be verbose: list all files found.
-    -n, --dry-run                   Collect files, but exit without processing and writing manifest file.
-    --include GLOB                  Search only files whose base name matches GLOB. [default: *.{wav,flac}].
-    -m METHOD, --method METHOD      Method for getting the true speech text of the audio files. [default: librispeech].
-    -e ADAT, --audio-env ADAT       Environment variable pointing to all audio data.
-    -o OUTPUT, --output OUTPUT      Output file name. [default: asr_manifest.json].
-
-Make a speech recognition (ASR) .json manifest file of all audio files under PATHS following the NVIDIA NeMo format.
-An example of manifest entries:
-
-{"audio_filepath": "<absolute_path_to>/1355-39947-0000.wav", "duration": 11.3, "text": "psychotherapy ..."}
-{"audio_filepath": "<absolute_path_to>/1355-39947-0001.wav", "duration": 15.905, "text": "it is an ..."}
-
-See the NVIDIA NeMo docs for more information:
-https://docs.nvidia.com/deeplearning/nemo/user-guide/docs/en/stable/asr/datasets.html
-
-Inputs:
-    PATH        A relative path name or list of paths containing audio files. Each will be
-                recursively searched for files matching the pattern GLOB.
-    GLOB        Match the pattern GLOB using wildcard matching.
-                Example: '*.{wav,flac}' matches all .wav and .flac files.
-    METHOD      The method to use for fetching the true speech of the audio files.
-                Supported methods:
-                    - 'librispeech'
-                    - 'vctk_noisy_speech' expects subdirs named like <name>_wav/ and <name>_txt/ with files in
-                      each using same basename, but with .wav and .txt respectively.
-                    - 'mcgill-speech' expects audio data in basename/speakerid/speakerid-promptid.wav and
-                      transcript data in Scripts/HarvardLists.dat
-    ADAT        Audio data environment variable. All found files will be expanded to their full, absolute path and
-                then parts of the path that match the specified environment variable value will be replaced with
-                the variable. This accommodates portability across platforms where the sound datasets may in
-                different locations.
-    OUTPUT      Name of output file. Default is asr_manifest.json.
-
-Outputs the following to the current directory:
-    <OUTPUT>
-    mkmanifest.log
-
-Example usage for LibriSpeech:
-  sonusai mkmanifest -mlibrispeech -eADAT -oasr_manifest.json --include='*.flac' train-clean-100
-  sonusai mkmanifest -m mcgill-speech -e ADAT -o asr_manifest_16k.json 16k-LP7/
-"""
-import signal
-
-
-def signal_handler(_sig, _frame):
-    import sys
-
-    from sonusai import logger
-
-    logger.info('Canceled due to keyboard interrupt')
-    sys.exit(1)
-
-
-signal.signal(signal.SIGINT, signal_handler)
-
-VALID_METHOD = ['librispeech', 'vctk_noisy_speech', 'mcgill-speech']
-
-
-def main() -> None:
-    from docopt import docopt
-
-    import sonusai
-    from sonusai.utils import trim_docstring
-
-    args = docopt(trim_docstring(__doc__), version=sonusai.__version__, options_first=True)
-
-    verbose = args['--verbose']
-    dry_run = args['--dry-run']
-    include = args['--include']
-    method = args['--method']
-    audio_env = args['--audio-env']
-    output = args['--output']
-    paths = args['PATH']
-
-    import json
-    from functools import partial
-    import time
-    from os import environ
-    from os.path import abspath
-    from os.path import join
-    from os.path import realpath
-
-    from tqdm import tqdm
-
-    from sonusai import SonusAIError
-    from sonusai import create_file_handler
-    from sonusai import initial_log_messages
-    from sonusai import logger
-    from sonusai import update_console_handler
-    from sonusai.utils import PathInfo
-    from sonusai.utils import braced_iglob
-    from sonusai.utils import pp_tqdm_imap
-    from sonusai.utils import seconds_to_hms
-    from sonusai.utils.asr_manifest_functions import collect_librispeech_transcripts
-    from sonusai.utils.asr_manifest_functions import collect_vctk_noisy_speech_transcripts
-    from sonusai.utils.asr_manifest_functions import get_librispeech_manifest_entry
-    from sonusai.utils.asr_manifest_functions import get_vctk_noisy_speech_manifest_entry
-    from sonusai.utils.asr_manifest_functions import get_mcgill_speech_manifest_entry
-
-    start_time = time.monotonic()
-
-    create_file_handler('mkmanifest.log')
-    update_console_handler(verbose)
-    initial_log_messages('mkmanifest')
-
-    if method not in VALID_METHOD:
-        raise SonusAIError(f'Unknown method: {method}')
-
-    audio_dir = None
-    if audio_env is not None:
-        audio_dir = realpath(environ[audio_env])
-        if audio_dir is None:
-            raise SonusAIError(f'Unknown environment variable: {audio_env}')
-
-    if audio_env:
-        for p in paths:
-            if not realpath(abspath(p)).startswith(audio_dir):
-                logger.warning(f'Specified directory, {p}, is not part of the provided audio environment: '
-                               f'${audio_env}={audio_dir}')
-
-    logger.info('')
-    logger.info(f'Searching {len(paths)} provided director{"ies" if len(paths) > 1 else "y"}...')
-
-    entries: list[PathInfo] = []
-    for p in paths:
-        location = join(realpath(abspath(p)), '**', include)
-        logger.debug(f'Processing {location}')
-        for file in braced_iglob(pathname=location, recursive=True):
-            name = file
-            if audio_env is not None:
-                name = name.replace(audio_dir, f'${audio_env}')
-            entries.append(PathInfo(abs_path=file, audio_filepath=name))
-        logger.debug('')
-
-    logger.info(f'Found {len(entries)} audio file{"s" if len(entries) != 1 else ""}')
-
-    if dry_run:
-        logger.info('')
-        logger.info('Dry run')
-        logger.info('')
-        for entry in entries:
-            logger.info(f' - {entry.audio_filepath}')
-        return
-
-    if method == 'librispeech':
-        logger.info('Collecting LibriSpeech transcript data')
-        transcript_data = collect_librispeech_transcripts(paths=paths)
-
-        processing_func = partial(get_librispeech_manifest_entry, transcript_data=transcript_data)
-        progress = tqdm(total=len(entries), desc='Creating LibriSpeech manifest data')
-        results = pp_tqdm_imap(processing_func, entries, progress=progress)
-        progress.close()
-
-        with open(output, 'w') as f:
-            for result in results:
-                f.write(json.dumps(result) + '\n')
-
-    if method == 'vctk_noisy_speech':
-        logger.info('Collecting VCTK Noisy Speech transcript data')
-        transcript_data = collect_vctk_noisy_speech_transcripts(paths=paths)
-
-        processing_func = partial(get_vctk_noisy_speech_manifest_entry, transcript_data=transcript_data)
-        progress = tqdm(total=len(entries), desc='Creating VCTK Noisy Speech manifest data')
-        results = pp_tqdm_imap(processing_func, entries, progress=progress)
-        progress.close()
-
-        with open(output, 'w') as f:
-            for result in results:
-                f.write(json.dumps(result) + '\n')
-
-    if method == 'mcgill-speech':
-        logger.info(f'Found {len(entries)} Mcgill Speech files, opening prompt file ...')
-        # Note expecting only one path pointing to data subdir
-        if len(paths) != 1:
-            raise SonusAIError(f'mcgill-speech only support a single path')
-        prompt_fpath = join(join(realpath(abspath(paths[0]))), '../Scripts/HarvardList.dat')
-        with open(prompt_fpath, encoding='utf-8') as f:
-            lines = f.readlines()
-
-        logger.info(f'Found {len(lines) - 4} entries in prompt file.')
-        # First 4 lines are header stuff, can use remaining directly with simple lookup
-        # example line: '01_02:Glue the sheet ...\n' (paragraph 1, sentence 2)
-        # 11 entries per group, so getting line is 11*(p1-1)+(s2-1)
-        lines = lines[4:]
-
-        processing_func = partial(get_mcgill_speech_manifest_entry, transcript_data=lines)
-        progress = tqdm(total=len(entries), desc='Creating Mcgill Speech manifest data')
-        results = pp_tqdm_imap(processing_func, entries, progress=progress)
-        progress.close()
-
-        with open(output, 'w') as f:
-            for result in results:
-                f.write(json.dumps(result) + '\n')
-
-    end_time = time.monotonic()
-    logger.info('')
-    logger.info(f'Completed in {seconds_to_hms(seconds=end_time - start_time)}')
-    logger.info('')
-
-
-if __name__ == '__main__':
-    main()

sonusai/utils/asr_manifest_functions/__init__.py

@@ -1,6 +0,0 @@
-from .data import TranscriptData
-from .librispeech import collect_librispeech_transcripts
-from .librispeech import get_librispeech_manifest_entry
-from .vctk_noisy_speech import collect_vctk_noisy_speech_transcripts
-from .vctk_noisy_speech import get_vctk_noisy_speech_manifest_entry
-from .mcgill_speech import get_mcgill_speech_manifest_entry

sonusai/utils/asr_manifest_functions/data.py

@@ -1 +0,0 @@
-TranscriptData = dict[str, str]

sonusai/utils/asr_manifest_functions/librispeech.py

@@ -1,46 +0,0 @@
-from sonusai.utils import PathInfo
-from sonusai.utils.asr_manifest_functions import TranscriptData
-
-
-def collect_librispeech_transcripts(paths: list[str] | str) -> TranscriptData:
-    from glob import iglob
-    from os.path import abspath
-    from os.path import dirname
-    from os.path import join
-
-    from sonusai import SonusAIError
-
-    entries: TranscriptData = {}
-    if not isinstance(paths, list):
-        paths = [paths]
-
-    for p in paths:
-        location = join(abspath(p), '**', '*.trans.txt')
-        for file in iglob(pathname=location, recursive=True):
-            root = dirname(file)
-            with open(file, encoding='utf-8') as f:
-                for line in f:
-                    name, text = line[: line.index(' ')], line[line.index(' ') + 1:]
-                    name = join(root, name)
-                    if name in entries:
-                        raise SonusAIError(f'{name} already exists in transcript data')
-                    entries[name] = text.lower().strip()
-    return entries
-
-
-def get_librispeech_manifest_entry(entry: PathInfo, transcript_data: TranscriptData) -> dict:
-    from os.path import splitext
-    from subprocess import check_output
-
-    from sonusai import SonusAIError
-
-    name = splitext(entry.abs_path)[0]
-    duration = float(check_output(f'soxi -D {entry.abs_path}', shell=True))
-    if name not in transcript_data.keys():
-        raise SonusAIError(f'Could not find {name} in transcript data')
-
-    return {
-        'audio_filepath': entry.audio_filepath,
-        'text':           transcript_data[name],
-        'duration':       duration,
-    }

sonusai/utils/asr_manifest_functions/mcgill_speech.py

@@ -1,29 +0,0 @@
-from sonusai.utils import PathInfo
-
-
-def get_mcgill_speech_manifest_entry(entry: PathInfo, transcript_data: list[str]) -> dict:
-    from os.path import splitext
-    from os.path import basename
-    from subprocess import check_output
-
-    from sonusai import SonusAIError
-
-    name = splitext(entry.abs_path)[0]
-    duration = float(check_output(f'soxi -D {entry.abs_path}', shell=True))
-    # i.e., from MA01_02.wav, get 01_02
-    promptname = basename(name)[2:]
-    # paragraph num
-    pnum = int(promptname[0:2])
-    snum = int(promptname[3:5])
-    idx = 11 * (pnum - 1) + (snum - 1)
-    try:
-        # remove prompt-id prefix and \n suffix
-        text = transcript_data[idx][6:-1]
-    except IndexError:
-        raise SonusAIError(f'Could not find {promptname}, idx {idx} in transcript data')
-
-    return {
-        'audio_filepath': entry.audio_filepath,
-        'text':           text,
-        'duration':       duration,
-    }

sonusai/utils/asr_manifest_functions/vctk_noisy_speech.py

@@ -1,66 +0,0 @@
-from sonusai.utils import PathInfo
-from sonusai.utils.asr_manifest_functions import TranscriptData
-
-
-def collect_vctk_noisy_speech_transcripts(paths: list[str] | str) -> TranscriptData:
-    from glob import iglob
-    from os import listdir
-    from os.path import abspath
-    from os.path import basename
-    from os.path import join
-    from os.path import split
-    from os.path import splitext
-
-    from sonusai import SonusAIError
-
-    entries: TranscriptData = {}
-    if not isinstance(paths, list):
-        paths = [paths]
-
-    for p in paths:
-        abs_p = abspath(p)
-        head, tail = split(abs_p)
-
-        dirs = listdir(head)
-        tail = tail.replace('wav', 'txt')
-
-        location = None
-        for d in dirs:
-            if tail.endswith(d):
-                location = join(head, d, '*.txt')
-                break
-        if location is None:
-            raise SonusAIError(f'Could not find VCTK Noisy Speech transcript data for {p}')
-
-        for file in iglob(pathname=location, recursive=True):
-            with open(file, encoding='utf-8') as f:
-                lines = f.readlines()
-                if len(lines) != 1:
-                    raise SonusAIError(f'Ill-formed VCTK Noisy Speech transcript file: {file}')
-
-                name = join(abs_p, splitext(basename(file))[0])
-                text = lines[0].lower().strip()
-
-                if name in entries:
-                    raise SonusAIError(f'{name} already exists in transcript data')
-                entries[name] = text.lower().strip()
-
-    return entries
-
-
-def get_vctk_noisy_speech_manifest_entry(entry: PathInfo, transcript_data: TranscriptData) -> dict:
-    from os.path import splitext
-    from subprocess import check_output
-
-    from sonusai import SonusAIError
-
-    name = splitext(entry.abs_path)[0]
-    duration = float(check_output(f'soxi -D {entry.abs_path}', shell=True))
-    if name not in transcript_data.keys():
-        raise SonusAIError(f'Could not find {name} in transcript data')
-
-    return {
-        'audio_filepath': entry.audio_filepath,
-        'text':           transcript_data[name],
-        'duration':       duration,
-    }