BatchalignHK 0.7.20.post14__tar.gz → 0.7.21__tar.gz

{batchalignhk-0.7.20.post14 → batchalignhk-0.7.21}/BatchalignHK.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: BatchalignHK
-Version: 0.7.20.post14
+Version: 0.7.21
 Summary: Python Speech Language Sample Analysis
 Author: Brian MacWhinney, Houjun Liu
 Author-email: macw@cmu.edu, houjun@cmu.edu
@@ -41,6 +41,7 @@ Requires-Dist: googletrans
 Requires-Dist: aliyun-python-sdk-core>=2.13.3
 Requires-Dist: oss2
 Requires-Dist: openai-whisper>=20240930
+Requires-Dist: funasr
 Provides-Extra: dev
 Requires-Dist: pytest; extra == "dev"
 Provides-Extra: train

{batchalignhk-0.7.20.post14 → batchalignhk-0.7.21}/BatchalignHK.egg-info/SOURCES.txt

@@ -92,6 +92,7 @@ batchalign/pipelines/analysis/__init__.py
 batchalign/pipelines/analysis/eval.py
 batchalign/pipelines/asr/__init__.py
 batchalign/pipelines/asr/aliyun.py
+batchalign/pipelines/asr/funaudio.py
 batchalign/pipelines/asr/num2chinese.py
 batchalign/pipelines/asr/oai_whisper.py
 batchalign/pipelines/asr/rev.py
@@ -125,6 +126,7 @@ batchalign/pipelines/translate/gtrans.py
 batchalign/pipelines/translate/seamless.py
 batchalign/pipelines/translate/utils.py
 batchalign/pipelines/utr/__init__.py
+batchalign/pipelines/utr/funaudio_utr.py
 batchalign/pipelines/utr/rev_utr.py
 batchalign/pipelines/utr/tencent_utr.py
 batchalign/pipelines/utr/utils.py

{batchalignhk-0.7.20.post14 → batchalignhk-0.7.21}/BatchalignHK.egg-info/requires.txt

@@ -31,6 +31,7 @@ googletrans
 aliyun-python-sdk-core>=2.13.3
 oss2
 openai-whisper>=20240930
+funasr
 
 [dev]
 pytest

{batchalignhk-0.7.20.post14 → batchalignhk-0.7.21}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: BatchalignHK
-Version: 0.7.20.post14
+Version: 0.7.21
 Summary: Python Speech Language Sample Analysis
 Author: Brian MacWhinney, Houjun Liu
 Author-email: macw@cmu.edu, houjun@cmu.edu
@@ -41,6 +41,7 @@ Requires-Dist: googletrans
 Requires-Dist: aliyun-python-sdk-core>=2.13.3
 Requires-Dist: oss2
 Requires-Dist: openai-whisper>=20240930
+Requires-Dist: funasr
 Provides-Extra: dev
 Requires-Dist: pytest; extra == "dev"
 Provides-Extra: train

{batchalignhk-0.7.20.post14 → batchalignhk-0.7.21}/batchalign/cli/cli.py

@@ -112,13 +112,15 @@ batchalign.add_command(train, "models")
               default=True, help="Use Whisper instead of Wav2Vec for English (defaults for Whisper for non-English)")
 @click.option("--tencent/--rev",
               default=False, help="Use Tencent instead of Rev.AI (default).")
+@click.option("--funaudio/--rev",
+              default=False, help="Use FunAudio instead of Rev.AI (default).")
 @click.option("--pauses", type=bool, default=False, help="Should we try to bullet each word or should we try to add pauses in between words by grouping them? Default: no pauses.", is_flag=True)
 @click.option("--wor/--nowor",
               default=True, help="Should we write word level alignment line? Default to yes.")
 @click.option("--data",
               help="the URL of the data", type=str)
 @click.pass_context
-def align(ctx, in_dir, out_dir, whisper, wav2vec, tencent, **kwargs):
+def align(ctx, in_dir, out_dir, whisper, wav2vec, tencent, funaudio, **kwargs):
     """Align transcripts against corresponding media files."""
     def loader(file):
         return (
@@ -135,8 +137,9 @@ def align(ctx, in_dir, out_dir, whisper, wav2vec, tencent, **kwargs):
                   in_dir, out_dir,
                   loader, writer, C,
                   fa="whisper_fa",
-                  utr=("whisper_utr" if whisper else
-                       ("tencent_utr" if tencent else "rev_utr")),
+                  utr = ("whisper_utr" if whisper else
+                         ("tencent_utr" if tencent else
+                         ("funaudio_utr" if funaudio else "rev_utr"))),
                   **kwargs)
     else:
         _dispatch("align", "eng", 1,
@@ -144,8 +147,9 @@ def align(ctx, in_dir, out_dir, whisper, wav2vec, tencent, **kwargs):
                   in_dir, out_dir,
                   loader, writer, C,
                   fa="wav2vec_fa",
-                  utr=("whisper_utr" if whisper else
-                       ("tencent_utr" if tencent else "rev_utr")),
+                  utr = ("whisper_utr" if whisper else
+                         ("tencent_utr" if tencent else
+                         ("funaudio_utr" if funaudio else "rev_utr"))),
                   **kwargs)
 
 #################### TRANSCRIBE ################################
@@ -162,6 +166,8 @@ def align(ctx, in_dir, out_dir, whisper, wav2vec, tencent, **kwargs):
               default=False, help="Use WhisperX instead of Rev.AI (default). Superceeds --whisper.")
 @click.option("--alibaba/--rev",
               default=False, help="Use Alibaba instead of Rev.AI (default). Superceeds --whisper.")
+@click.option("--funaudio/--rev",
+              default=False, help="Use FunAudio instead of Rev.AI (default). Superceeds --whisper.")
 @click.option("--diarize/--nodiarize",
               default=False, help="Perform speaker diarization (this flag is ignored with Rev.AI)")
 @click.option("--wor/--nowor",
@@ -192,6 +198,8 @@ def transcribe(ctx, in_dir, out_dir, lang, num_speakers, **kwargs):
         asr = "whisper_oai"
     if kwargs["alibaba"]:
         asr = "aliyun"
+    if kwargs["funaudio"]:
+        asr = "funaudio"
 
     def writer(doc, output):
         doc.content.insert(0, CustomLine(id="Comment", type=CustomLineType.INDEPENDENT,
@@ -337,6 +345,8 @@ def utseg(ctx, in_dir, out_dir, lang, num_speakers, **kwargs):
               default=False, help="Use OpenAI Whisper (ASR) instead of Rev.AI (default).")
 @click.option("--tencent/--rev",
               default=False, help="Use Tencent instead of Rev.AI (default).")
+@click.option("--funaudio/--rev",
+              default=False, help="Use Tencent instead of Rev.AI (default).")
 @click.option("--lang",
               help="sample language in three-letter ISO 3166-1 alpha-3 code",
               show_default=True,
@@ -346,8 +356,6 @@ def utseg(ctx, in_dir, out_dir, lang, num_speakers, **kwargs):
               help="the URL of the data",
               type=str)
 @click.option("-n", "--num_speakers", type=int, help="number of speakers in the language sample", default=2)
-@click.option("--wor/--nowor",
-              default=False, help="Should we write word level alignment line? Default to no.")
 @click.pass_context
 def benchmark(ctx, in_dir, out_dir, lang, num_speakers, whisper, tencent, **kwargs):
     """Benchmark ASR utilities for their word accuracy"""
@@ -371,14 +379,13 @@ def benchmark(ctx, in_dir, out_dir, lang, num_speakers, whisper, tencent, **kwar
             df.write(str(doc["wer"]))
         with open(Path(output).with_suffix(".diff"), 'w') as df:
             df.write(str(doc["diff"]))
-        CHATFile(doc=doc["doc"]).write(str(Path(output).with_suffix(".asr.cha")),
-                                       write_wor=kwargs.get("wor", False))
+        CHATFile(doc=doc["doc"]).write(str(Path(output).with_suffix(".asr.cha")))
 
 
     _dispatch("benchmark", lang, num_speakers, ["mp3", "mp4", "wav"], ctx,
               in_dir, out_dir,
               loader, writer, C,
-              asr="whisper" if whisper else ("tencent" if tencent else "rev"), **kwargs)
+              asr="whisper" if whisper else ("funaudio" if funaudio else ("tencent" if tencent else "rev")), **kwargs)
 
 
 #################### SETUP ################################

{batchalignhk-0.7.20.post14 → batchalignhk-0.7.21}/batchalign/pipelines/__init__.py

@@ -1,16 +1,16 @@
 from .pipeline import BatchalignPipeline
 from .base import BatchalignEngine
 from .asr import (WhisperEngine, RevEngine, WhisperXEngine,
-                  TencentEngine, OAIWhisperEngine, AliyunEngine)
+                  TencentEngine, OAIWhisperEngine, AliyunEngine, FunAudioEngine)
 
 from .morphosyntax import StanzaEngine, CorefEngine
 from .cleanup import NgramRetraceEngine, DisfluencyReplacementEngine
 from .speaker import NemoSpeakerEngine
 
 from .fa import WhisperFAEngine, Wave2VecFAEngine
-from .utr import WhisperUTREngine, RevUTREngine, TencentUTREngine
+from .utr import WhisperUTREngine, RevUTREngine, TencentUTREngine, FunAudioUTREngine
 
 from .analysis import EvaluationEngine
 from .utterance import StanzaUtteranceEngine
 
-from .translate import SeamlessTranslationModel, GoogleTranslateEngine
+# from .translate import SeamlessTranslationModel, GoogleTranslateEngine

{batchalignhk-0.7.20.post14 → batchalignhk-0.7.21}/batchalign/pipelines/asr/__init__.py

@@ -4,3 +4,4 @@ from .whisperx import WhisperXEngine
 from .tencent import TencentEngine
 from .oai_whisper import OAIWhisperEngine
 from .aliyun import AliyunEngine
+from .funaudio import FunAudioEngine

batchalignhk-0.7.21/batchalign/pipelines/asr/funaudio.py

@@ -0,0 +1,231 @@
+"""
+rev.py
+Support for Rev.ai, a commerical ASR service
+"""
+
+from batchalign.document import *
+from batchalign.pipelines.base import *
+from batchalign.pipelines.asr.utils import *
+from batchalign.utils.config import config_read
+
+from batchalign.errors import *
+
+from batchalign.models import BertUtteranceModel, BertCantoneseUtteranceModel, resolve
+
+from opencc import OpenCC
+cc = OpenCC('s2hk')
+
+import time
+import pathlib
+import tempfile
+import pycountry
+import numpy as np
+import soundfile as sf
+# from pydub import AudioSegment
+# from pydub.effects import normalize
+import base64
+from tencentcloud.common.credential import Credential
+from tencentcloud.asr.v20190614.asr_client import AsrClient, models
+
+import asyncio
+import tempfile
+import os
+# from pydub import AudioSegment
+# from pydub.effects import normalize
+# from pydub.exceptions import CouldntDecodeError
+from funasr import AutoModel
+from funasr.utils.postprocess_utils import rich_transcription_postprocess
+
+import logging
+L = logging.getLogger("batchalign")
+
+class FunAudioEngine(BatchalignEngine):
+
+    @property
+    def tasks(self):
+        if self.__engine:
+            return [ Task.ASR, Task.UTTERANCE_SEGMENTATION ]
+        else:
+            return [ Task.ASR ]
+        
+    def __init__(self, model="FunAudioLLM/SenseVoiceSmall", lang="yue"):
+
+        self.model_dir = model
+        self.__lang = "yue"
+        
+        self.model = AutoModel(
+            model=self.model_dir,
+            output_timestamps=True,
+            vad_model="fsmn-vad",
+            vad_kwargs={"max_single_segment_time": 30000},
+            device="cuda:0",  # GPU
+            hub="hf",
+            cache={},
+            language="yue", 
+            use_itn=True,
+            batch_size_s=60,
+            output_timestamp=True,
+            ban_emo_unk =False,
+            merge_vad=True,
+            merge_length_s=15,
+           
+        )
+
+        if resolve("utterance", self.__lang) != None:
+            L.debug("Initializing utterance model...")
+            if lang != "yue":
+                self.__engine = BertUtteranceModel(resolve("utterance", lang))
+            else:
+                # we have special inference procedure for cantonese
+                self.__engine = BertCantoneseUtteranceModel(resolve("utterance", lang))
+            L.debug("Done.")
+        else:
+            self.__engine = None
+            
+    def replace_cantonese_words(self, text):
+        """Function to replace Cantonese words with custom replacements."""
+        word_replacements = {
+            "系": "係", 
+            "繫": "係",
+            "聯係": "聯繫",
+            "系啊": "係啊",
+            "真系": "真係",
+            "唔系": "唔係",
+            "呀": "啊",
+            "噶": "㗎",
+            "咧": "呢",
+            "嗬": "喎",
+            "只": "隻",
+            "咯": "囉",
+            "嚇": "吓",
+            "飲": "飲",
+            "喐": "郁",
+            "食": "食",
+            "啫": "咋",
+            "哇": "嘩",
+            "着": "著",
+            "中意": "鍾意",
+            "嘞": "喇",
+            "啵": "噃",
+            "遊水": "游水",
+            "羣組": "群組",
+            "古仔": "故仔",
+            "甕": "㧬",
+            "牀": "床",
+            "松": "鬆",
+            "較剪": "鉸剪",
+            "吵": "嘈",
+            "衝涼": "沖涼",
+            "分鍾": "分鐘",
+            "重復": "重複"
+        }
+        sorted_keys = sorted(word_replacements.keys(), key=len, reverse=True)
+        pattern = re.compile('|'.join(re.escape(key) for key in sorted_keys))
+        
+        def replace_word(match):
+            matched_text = match.group(0)  # Extract the matched word
+            return word_replacements.get(matched_text, matched_text)  # Replace or return the original word
+
+        return pattern.sub(replace_word, text)
+
+    @staticmethod
+    def is_roman(x):
+        """check if x contains only roman characters"""
+        return all(c.isalpha() and ord(c) < 128 for c in x if not c.isspace())
+    
+    def generate(self, audio_file_path):
+        """
+        Generate transcription from an audio file using the FunAudio model.
+        :param audio_file_path: Path to the audio file to be transcribed.
+        :return: A Document object containing the transcription and metadata.
+        """
+        res = self.model.generate(
+            input=audio_file_path,
+            cache={},
+            language=self.__lang, 
+            output_timestamps=True,
+            vad_model="fsmn-vad",
+            vad_kwargs={"max_single_segment_time": 60000},
+            ban_emo_unk=False,
+            use_itn=True,
+            batch_size_s=60,
+            merge_vad=True,
+            merge_length_s=15,
+            output_timestamp=True,
+            spk_model="cam++"
+        )
+        
+        turns = []
+
+        for segment in res:  # segment is a dictionary with keys "text" and "timestamp"
+            print("segment:", segment)
+            print(type(segment))
+
+            # Extracting text and timestamps from the segment
+            text = segment["text"]
+            print(text)
+            timestamps = segment["timestamp"]
+
+            # Check if timestamps is a list of tuples
+            utterances = []
+            current_utterance = []
+            for part in text.split("<|yue|>"):
+                if not part.strip():
+                    continue
+                parts = part.strip().split("<|withitn|>", 1)
+                if len(parts) > 1:
+                    emotion = parts[0].strip()
+                    content = parts[1].strip()
+             
+                    current_utterance.append(content)
+                    print(f"current_utterance:{current_utterance}")
+
+            large_string = ''.join(current_utterance)
+            print(f"Large string: {large_string}")
+        
+            turn = []
+                
+            # process Cantonese differently
+            if self.__lang == "yue":
+                content = cc.convert(large_string)
+                content = self.replace_cantonese_words(content)
+                content = content.replace("「", "").replace("」", "").replace("。", "").replace("，", "").replace("！", "").replace("？", "")
+                print(f"Processed Cantonese content: {content}")
+
+               
+                items = list(content)  
+            else:                    
+                items = large_string.split()
+                
+            turn = []
+
+            num_items = len(items)  
+            print("Number of items:", num_items)              
+            for index, item in enumerate(items):
+                print(f"Processing item {index + 1}/{num_items}: {item}")
+                item_start, item_end = timestamps[index]
+                
+
+                turn.append({
+                    "type": "text",
+                    "ts": item_start / 1000,
+                    "end_ts": item_end / 1000,
+                    "value": item  
+                })
+
+
+            turns.append({
+                "elements": turn,
+                "speaker": 0
+            })
+
+        L.debug(f"Funaudio done.")
+
+        doc = process_generation({"monologues": turns},
+                                self.__lang, 
+                                utterance_engine=self.__engine)
+        media = Media(type=MediaType.AUDIO, name=Path(audio_file_path).stem, url=audio_file_path)
+        doc.media = media
+        return doc
+
+

{batchalignhk-0.7.20.post14 → batchalignhk-0.7.21}/batchalign/pipelines/asr/utils.py

@@ -163,14 +163,16 @@ def process_generation(output, lang="eng", utterance_engine=None):
     for utterance in output["monologues"]:
         # get a list of words
         words = merge_on_wordlist(utterance["elements"])
+        # words = utterance["elements"]
         # coallate words (not punct) into the shape we expect
         # which is ['word', [start_ms, end_ms]]. Yes, this would
         # involve multiplying by 1000 to s => ms
         words = [[i["value"], [round(i["ts"]*1000) if i.get("ts") != None else None,
-                               round(i["end_ts"]*1000) if i.get("end_ts") != None else the]] # None shape
+                                round(i["end_ts"]*1000) if i.get("end_ts") != None else None]] # the shape
                 for i in words # for each word
                     if i["value"].strip() != "" and
-                 not re.match(r'<.*>', i["value"])] # if its text (i.e. not "pause")
+                    not re.match(r'<.*>', i["value"])] # if its text (i.e. not "pause")
+
 
         # sometimes, the system outputs two forms with a space as one single
         # word. we need to interpolate the space between them
@@ -188,6 +190,7 @@ def process_generation(output, lang="eng", utterance_engine=None):
             # if we only have one part, we don't interpolate
             if len(word_parts) == 1:
                 final_words.append([word, [i,o]])
+                words = merge_on_wordlist(utterance["elements"])
                 continue
             # otherwise, we interpolate the itme
             cur = i

{batchalignhk-0.7.20.post14 → batchalignhk-0.7.21}/batchalign/pipelines/dispatch.py

@@ -6,8 +6,8 @@ Tabulate default packages and options.
 from batchalign import (WhisperEngine, WhisperFAEngine, StanzaEngine, RevEngine,
                         NgramRetraceEngine, DisfluencyReplacementEngine, WhisperUTREngine,
                         RevUTREngine, EvaluationEngine, WhisperXEngine, NemoSpeakerEngine,
-                        StanzaUtteranceEngine, CorefEngine, Wave2VecFAEngine, SeamlessTranslationModel, TencentEngine,
-                        GoogleTranslateEngine, OAIWhisperEngine, TencentUTREngine, AliyunEngine)
+                        StanzaUtteranceEngine, CorefEngine, Wave2VecFAEngine, TencentEngine,
+                        OAIWhisperEngine, TencentUTREngine, AliyunEngine, FunAudioEngine, FunAudioUTREngine)
 
 from batchalign import BatchalignPipeline
 from batchalign.models import resolve
@@ -144,6 +144,10 @@ def dispatch_pipeline(pkg_str, lang, num_speakers=None, **arg_overrides):
             engines.append(OAIWhisperEngine())
         elif engine == "aliyun":
             engines.append(AliyunEngine())
+        elif engine == "funaudio":
+            engines.append(FunAudioEngine())
+        elif engine ==  "funaudio_utr":
+            engines.append(FunAudioUTREngine(lang=lang))
 
 
     L.debug(f"Done initalizing packages.")

{batchalignhk-0.7.20.post14 → batchalignhk-0.7.21}/batchalign/pipelines/morphosyntax/ud.py

@@ -884,7 +884,7 @@ def morphoanalyze(doc: Document, retokenize:bool, skipmultilang:bool, status_hoo
 
             # parse the stanza output
             mor, gra = parse_sentence(sents[0], ending, special_forms_cleaned, lang[0])
-            mor = mor.replace("~part|s verb|", "~aux|is verb|")
+            mor = re.sub(r"~part\|s verb\|(\w+)-Ger-S", r"~aux|is verb|\1-Part-Pres-S", mor)
             # breakpoint()
 
             if mor.strip() == "" or mor.strip() in ENDING_PUNCT:

{batchalignhk-0.7.20.post14 → batchalignhk-0.7.21}/batchalign/pipelines/pipeline.py

@@ -109,7 +109,8 @@ class BatchalignPipeline:
             L.debug(f"Calling generator: {self.__generator}")
             if callback:
                 callback(0,total_tasks, self.__generator.tasks)
-            doc = self.__generator.generate(doc.media.url, **kwargs)
+            
+            doc = self.__generator.generate(doc.media.url)
             if callback:
                 callback(1,total_tasks, self.__generator.tasks)
 

{batchalignhk-0.7.20.post14 → batchalignhk-0.7.21}/batchalign/pipelines/utr/__init__.py

@@ -1,3 +1,4 @@
 from .whisper_utr import WhisperUTREngine
 from .rev_utr import RevUTREngine
 from .tencent_utr import TencentUTREngine
+from .funaudio_utr import FunAudioUTREngine

batchalignhk-0.7.21/batchalign/pipelines/utr/funaudio_utr.py

@@ -0,0 +1,76 @@
+import os
+from batchalign.document import *
+from batchalign.pipelines.base import *
+from batchalign.pipelines.asr.utils import *
+from batchalign.pipelines.utr.utils import bulletize_doc
+from batchalign.pipelines.asr.funaudio import FunAudioEngine
+
+from opencc import OpenCC
+cc = OpenCC('s2hk')
+
+import warnings 
+
+import pycountry
+
+import logging
+L = logging.getLogger("batchalign")
+
+class FunAudioUTREngine(BatchalignEngine):
+    tasks = [ Task.UTTERANCE_TIMING_RECOVERY ]
+
+    def __init__(self, model=None, lang="yue"):
+        model = "FunAudioLLM/SenseVoiceSmall"
+
+        language = pycountry.languages.get(alpha_3=lang).name
+            
+        self.__funaudio = FunAudioEngine(model, lang="yue")
+        self.__lang = lang
+
+    def process(self, doc, **kwargs):
+        # bring language code into the stack to access
+        lang = doc.langs[0]
+
+        # check and if there are existing utterance timings, warn
+        if any([i.alignment for i in doc.content if isinstance(i, Utterance)]):
+            warnings.warn(f"We found existing utterance timings in the document with {doc.media.url}! Skipping rough utterance alignment.")
+            return doc
+
+        f = kwargs.get("extra_info", {}).get("extra_input")
+
+        if not f:
+            assert doc.media != None and doc.media.url != None, f"We cannot add utterance timings to something that doesn't have a media path! Provided media tier='{doc.media}'"
+
+        f = f if f else doc.media.url
+        
+        res = self.__funaudio.generate(
+            audio_file_path=doc.media.url
+        )
+
+        element_lit = res.content
+        res_content = []
+        for item in element_lit:
+            text = item[0]
+            text = str(text)
+            match = re.search(r"text='(.*?)'.*time=\((\d+),\s*(\d+)\)", text)
+            if match:
+                text = match.group(1) 
+                start_ms = int(match.group(2)) 
+                end_ms = int(match.group(3)) 
+                print(type(end_ms))
+            
+            res_content.append({
+            "value": text,
+            "ts": start_ms / 1000.0,  
+            "end_ts": end_ms / 1000.0  
+            })
+
+            
+            turns = []
+            turns.append({
+                "elements": res_content,
+                "speaker": "unknown"  
+            })
+            
+            element = {"monologues": turns}           
+            
+        return bulletize_doc(element, doc)

batchalignhk-0.7.21/batchalign/version

@@ -0,0 +1,3 @@
+0.7.21
+August 05, 2025
+FunAudio ASR

{batchalignhk-0.7.20.post14 → batchalignhk-0.7.21}/setup.py

@@ -62,7 +62,8 @@ setup(
         "googletrans",
         "aliyun-python-sdk-core>=2.13.3",
         "oss2",
-        "openai-whisper>=20240930"
+        "openai-whisper>=20240930",
+        "funasr"
     ],
     extras_require={
         'dev': [

batchalignhk-0.7.20.post14/batchalign/version

@@ -1,3 +0,0 @@
-0.7.20-post.14
-August 05, 2025
-benchmarking fixes