BatchalignHK 0.7.17.post22__tar.gz → 0.7.18b0__tar.gz

{batchalignhk-0.7.17.post22 → batchalignhk-0.7.18b0}/BatchalignHK.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: BatchalignHK
-Version: 0.7.17.post22
+Version: 0.7.18b0
 Summary: Python Speech Language Sample Analysis
 Author: Brian MacWhinney, Houjun Liu
 Author-email: macw@cmu.edu, houjun@cmu.edu
@@ -38,7 +38,7 @@ Requires-Dist: tiktoken
 Requires-Dist: blobfile
 Requires-Dist: sentencepiece
 Requires-Dist: tencentcloud-sdk-python-common
-Requires-Dist: tencentcloud-sdk-python-asr
+Requires-Dist: tencentcloud-sdk-python-asrgoogletrans
 Provides-Extra: dev
 Requires-Dist: pytest; extra == "dev"
 Provides-Extra: train

{batchalignhk-0.7.17.post22 → batchalignhk-0.7.18b0}/BatchalignHK.egg-info/SOURCES.txt

@@ -86,7 +86,9 @@ batchalign/pipelines/morphosyntax/ja/verbforms.py
 batchalign/pipelines/speaker/__init__.py
 batchalign/pipelines/speaker/nemo_speaker.py
 batchalign/pipelines/translate/__init__.py
+batchalign/pipelines/translate/gtrans.py
 batchalign/pipelines/translate/seamless.py
+batchalign/pipelines/translate/utils.py
 batchalign/pipelines/utr/__init__.py
 batchalign/pipelines/utr/rev_utr.py
 batchalign/pipelines/utr/utils.py

{batchalignhk-0.7.17.post22 → batchalignhk-0.7.18b0}/BatchalignHK.egg-info/requires.txt

@@ -28,7 +28,7 @@ tiktoken
 blobfile
 sentencepiece
 tencentcloud-sdk-python-common
-tencentcloud-sdk-python-asr
+tencentcloud-sdk-python-asrgoogletrans
 
 [dev]
 pytest

{batchalignhk-0.7.17.post22 → batchalignhk-0.7.18b0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: BatchalignHK
-Version: 0.7.17.post22
+Version: 0.7.18b0
 Summary: Python Speech Language Sample Analysis
 Author: Brian MacWhinney, Houjun Liu
 Author-email: macw@cmu.edu, houjun@cmu.edu
@@ -38,7 +38,7 @@ Requires-Dist: tiktoken
 Requires-Dist: blobfile
 Requires-Dist: sentencepiece
 Requires-Dist: tencentcloud-sdk-python-common
-Requires-Dist: tencentcloud-sdk-python-asr
+Requires-Dist: tencentcloud-sdk-python-asrgoogletrans
 Provides-Extra: dev
 Requires-Dist: pytest; extra == "dev"
 Provides-Extra: train

{batchalignhk-0.7.17.post22 → batchalignhk-0.7.18b0}/batchalign/cli/cli.py

@@ -315,6 +315,8 @@ def utseg(ctx, in_dir, out_dir, lang, num_speakers, **kwargs):
 @common_options
 @click.option("--whisper/--rev",
               default=False, help="Use OpenAI Whisper (ASR) instead of Rev.AI (default).")
+@click.option("--tencent/--rev",
+              default=False, help="Use Tencent instead of Rev.AI (default).")
 @click.option("--lang",
               help="sample language in three-letter ISO 3166-1 alpha-3 code",
               show_default=True,
@@ -322,7 +324,7 @@ def utseg(ctx, in_dir, out_dir, lang, num_speakers, **kwargs):
               type=str)
 @click.option("-n", "--num_speakers", type=int, help="number of speakers in the language sample", default=2)
 @click.pass_context
-def benchmark(ctx, in_dir, out_dir, lang, num_speakers, whisper, **kwargs):
+def benchmark(ctx, in_dir, out_dir, lang, num_speakers, whisper, tencent, **kwargs):
     """Benchmark ASR utilities for their word accuracy"""
     def loader(file):
         # try to find a .cha in the same directory
@@ -348,7 +350,7 @@ def benchmark(ctx, in_dir, out_dir, lang, num_speakers, whisper, **kwargs):
     _dispatch("benchmark", lang, num_speakers, ["mp3", "mp4", "wav"], ctx,
               in_dir, out_dir,
               loader, writer, C,
-              asr="whisper" if whisper else "rev", **kwargs)
+              asr="whisper" if whisper else ("tencent" if tencent else "rev"), **kwargs)
 
 
 #################### SETUP ################################

{batchalignhk-0.7.17.post22 → batchalignhk-0.7.18b0}/batchalign/formats/chat/generator.py

@@ -30,8 +30,8 @@ def generate_chat_utterance(utterance: Utterance, special_mor=False, write_wor=T
     main_line = str(utterance)
     # last minut ecorrections
     main_line = re.sub(r"<([\w ]+) \[\/", r"<\1> [/", main_line)
-    main_line = re.sub(r"«", "\"", main_line)
-    main_line = re.sub(r"»", "\"", main_line)
+    main_line = re.sub(r"«", "“", main_line)
+    main_line = re.sub(r"»", "”", main_line)
     main_line = re.sub(r"—", "-", main_line)
     main_line = re.sub(r"–", "-", main_line)
     tier = utterance.tier

{batchalignhk-0.7.17.post22 → batchalignhk-0.7.18b0}/batchalign/pipelines/__init__.py

@@ -12,4 +12,4 @@ from .utr import WhisperUTREngine, RevUTREngine
 from .analysis import EvaluationEngine
 from .utterance import StanzaUtteranceEngine
 
-from .translate import SeamlessTranslationModel
+from .translate import SeamlessTranslationModel, GoogleTranslateEngine

{batchalignhk-0.7.17.post22 → batchalignhk-0.7.18b0}/batchalign/pipelines/dispatch.py

@@ -6,7 +6,9 @@ Tabulate default packages and options.
 from batchalign import (WhisperEngine, WhisperFAEngine, StanzaEngine, RevEngine,
                         NgramRetraceEngine, DisfluencyReplacementEngine, WhisperUTREngine,
                         RevUTREngine, EvaluationEngine, WhisperXEngine, NemoSpeakerEngine,
-                        StanzaUtteranceEngine, CorefEngine, Wave2VecFAEngine, SeamlessTranslationModel, TencentEngine)
+                        StanzaUtteranceEngine, CorefEngine, Wave2VecFAEngine, SeamlessTranslationModel, TencentEngine,
+                        GoogleTranslateEngine)
+
 from batchalign import BatchalignPipeline
 from batchalign.models import resolve
 
@@ -28,7 +30,7 @@ DEFAULT_PACKAGES = {
     "eval": "evaluation",
     "utterance": "stanza_utt",
     "coref": "stanza_coref",
-    "translate": "seamless_translate",
+    "translate": "gtrans",
 }
 
 LANGUAGE_OVERRIDE_PACKAGES = {
@@ -134,6 +136,8 @@ def dispatch_pipeline(pkg_str, lang, num_speakers=None, **arg_overrides):
             engines.append(SeamlessTranslationModel())
         elif engine == "tencent":
             engines.append(TencentEngine(lang=lang))
+        elif engine == "gtrans":
+            engines.append(GoogleTranslateEngine())
 
 
     L.debug(f"Done initalizing packages.")

{batchalignhk-0.7.17.post22 → batchalignhk-0.7.18b0}/batchalign/pipelines/translate/__init__.py

@@ -1 +1,2 @@
 from .seamless import SeamlessTranslationModel
+from .gtrans import GoogleTranslateEngine

batchalignhk-0.7.18b0/batchalign/pipelines/translate/gtrans.py

@@ -0,0 +1,58 @@
+from batchalign.models import WhisperFAModel
+from batchalign.document import *
+from batchalign.pipelines.base import *
+from batchalign.utils import *
+from batchalign.utils.dp import *
+from batchalign.constants import *
+from batchalign.pipelines.translate.utils import run_coroutine_sync
+
+from googletrans import Translator
+
+import logging
+L = logging.getLogger("batchalign")
+
+import re
+
+# !uv pip install sentencepiece
+
+import pycountry
+import warnings
+import time
+
+import asyncio
+
+class GoogleTranslateEngine(BatchalignEngine):
+    tasks = [ Task.TRANSLATE ]
+
+    def _hook_status(self, status_hook):
+        self.status_hook = status_hook
+
+    def __init__(self):
+        self.status_hook = None
+
+    async def translate(self, text):
+        translator = Translator()
+        return await translator.translate(text)
+
+    def process(self, doc:Document, **kwargs):
+
+        for indx, i in enumerate(doc.content):
+            if not isinstance(i, Utterance):
+                continue
+            if i.translation:
+                continue
+            
+            text = i.strip(join_with_spaces=False, include_retrace=True, include_fp=True)
+            translated_text_from_text = run_coroutine_sync(self.translate(text)).text
+
+            i.translation = translated_text_from_text
+            for j in MOR_PUNCT + ENDING_PUNCT:
+                i.translation = i.translation.replace(j, " "+j)
+
+            if self.status_hook != None:
+                self.status_hook(indx+1, len(doc.content))
+            time.sleep(1.5)
+
+        return doc
+
+

batchalignhk-0.7.18b0/batchalign/pipelines/translate/utils.py

@@ -0,0 +1,35 @@
+import asyncio
+import threading
+from concurrent.futures import ThreadPoolExecutor
+from typing import Any, Coroutine, TypeVar
+
+__all__ = [
+    "run_coroutine_sync",
+]
+
+T = TypeVar("T")
+
+
+def run_coroutine_sync(coroutine: Coroutine[Any, Any, T], timeout: float = 30) -> T:
+    def run_in_new_loop():
+        new_loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(new_loop)
+        try:
+            return new_loop.run_until_complete(coroutine)
+        finally:
+            new_loop.close()
+
+    try:
+        loop = asyncio.get_running_loop()
+    except RuntimeError:
+        return asyncio.run(coroutine)
+
+    if threading.current_thread() is threading.main_thread():
+        if not loop.is_running():
+            return loop.run_until_complete(coroutine)
+        else:
+            with ThreadPoolExecutor() as pool:
+                future = pool.submit(run_in_new_loop)
+                return future.result(timeout=timeout)
+    else:
+        return asyncio.run_coroutine_threadsafe(coroutine, loop).result()

batchalignhk-0.7.18b0/batchalign/version

@@ -0,0 +1,3 @@
+0.7.18-beta.0
+April 16th, 2025
+google translate

{batchalignhk-0.7.17.post22 → batchalignhk-0.7.18b0}/setup.py

@@ -58,6 +58,7 @@ setup(
         "sentencepiece",
         "tencentcloud-sdk-python-common",
         "tencentcloud-sdk-python-asr"
+        "googletrans"
     ],
     extras_require={
         'dev': [

batchalignhk-0.7.17.post22/batchalign/version

@@ -1,3 +0,0 @@
-0.7.17-post.22
-April 16th, 2025
-Various fixes for numbers