BatchalignHK 0.7.19.post9__tar.gz → 0.7.19.post11__tar.gz

{batchalignhk-0.7.19.post9 → batchalignhk-0.7.19.post11}/BatchalignHK.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: BatchalignHK
-Version: 0.7.19.post9
+Version: 0.7.19.post11
 Summary: Python Speech Language Sample Analysis
 Author: Brian MacWhinney, Houjun Liu
 Author-email: macw@cmu.edu, houjun@cmu.edu

{batchalignhk-0.7.19.post9 → batchalignhk-0.7.19.post11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: BatchalignHK
-Version: 0.7.19.post9
+Version: 0.7.19.post11
 Summary: Python Speech Language Sample Analysis
 Author: Brian MacWhinney, Houjun Liu
 Author-email: macw@cmu.edu, houjun@cmu.edu

{batchalignhk-0.7.19.post9 → batchalignhk-0.7.19.post11}/batchalign/pipelines/asr/tencent.py

@@ -111,6 +111,11 @@ class TencentEngine(BatchalignEngine):
             "重復": "重複"
         }
         return word_replacements.get(word, word)
+
+    @staticmethod
+    def is_roman(x):
+        """check if x contains only roman characters"""
+        return all(c.isalpha() and ord(c) < 128 for c in x if not c.isspace())
     
     def generate(self, f, **kwargs):
         lang = self.__lang
@@ -119,73 +124,95 @@ class TencentEngine(BatchalignEngine):
         # processed_path = self.__preprocess_audio(f)
         # audio = AudioSegment.from_file(processed_path)
         
-        try:
-            L.info(f"Uploading '{pathlib.Path(f).stem}'...")
-            # we will send the file for processing
-            if not str(f).startswith("http"):
-                with open(f, "rb") as image_file:
-                    encoded_string = base64.b64encode(image_file.read())
-
-            req = models.CreateRecTaskRequest()
-            if lang in {'zho', 'yue', 'wuu', 'nan','hak'}:
-                req.EngineModelType = "16k_zh_large"
-            else:
-                req.EngineModelType = f"16k_{lang}"
-            req.ResTextFormat = 1
-            req.SpeakerDiarization = 1
-            req.ChannelNum = 1
-            if not str(f).startswith("http"):
-                req.Data = encoded_string.decode('ascii')
-                req.SourceType = 1
-            else:
-                req.Url = f
-                req.SourceType = 0
-            resp = client.CreateRecTask(req)
+        L.info(f"Uploading '{pathlib.Path(f).stem}'...")
+        # we will send the file for processing
+        if not str(f).startswith("http"):
+            with open(f, "rb") as image_file:
+                encoded_string = base64.b64encode(image_file.read())
+
+        req = models.CreateRecTaskRequest()
+        if lang in {'zho', 'yue', 'wuu', 'nan','hak'}:
+            req.EngineModelType = "16k_zh_large"
+        else:
+            req.EngineModelType = f"16k_{lang}"
+        req.ResTextFormat = 1
+        req.SpeakerDiarization = 1
+        req.ChannelNum = 1
+        if not str(f).startswith("http"):
+            req.Data = encoded_string.decode('ascii')
+            req.SourceType = 1
+        else:
+            req.Url = f
+            req.SourceType = 0
+        resp = client.CreateRecTask(req)
 
-            L.info(f"Tencent is transcribing '{pathlib.Path(f).stem}'...")
-            req = models.DescribeTaskStatusRequest()
-            req.TaskId = resp.Data.TaskId
+        L.info(f"Tencent is transcribing '{pathlib.Path(f).stem}'...")
+        req = models.DescribeTaskStatusRequest()
+        req.TaskId = resp.Data.TaskId
 
+        res = client.DescribeTaskStatus(req)
+        while res.Data.Status not in [2, 3]:
+            time.sleep(15)
             res = client.DescribeTaskStatus(req)
-            while res.Data.Status not in [2, 3]:
-                time.sleep(15)
-                res = client.DescribeTaskStatus(req)
-
-            if res.Data.Status in ["3", 3]:
-                raise RuntimeError(f"Tencent reports job failed! error='{res.Data.ErrorMsg}'")
-
-            turns = []
-            for i in res.Data.ResultDetail:
-                turn = []
-                start = i.StartMs
-                for j in i.Words:
-                    word = j.Word
-                    if self.__lang == "yue":
-                        word = cc.convert(word)
-                        
-                        word = self.replace_cantonese_words(word)
 
+        if res.Data.Status in ["3", 3]:
+            raise RuntimeError(f"Tencent reports job failed! error='{res.Data.ErrorMsg}'")
+
+        turns = []
+        for i in res.Data.ResultDetail:
+            turn = []
+            start = i.StartMs
+            roman_cache = ""
+            roman_cache_start = i.StartMs
+            roman_cache_end = i.StartMs
+            for j in i.Words:
+                word = j.Word
+                if self.__lang == "yue":
+                    word = cc.convert(word)
+
+                    word = self.replace_cantonese_words(word)
+
+                if self.is_roman(word):
+                    if roman_cache == "":
+                        roman_cache_start = (j.OffsetStartMs + start)
+                    roman_cache = roman_cache + word
+                    roman_cache_end = (j.OffsetEndMs + start)
+                else:
+                    if roman_cache != "":
+                        turn.append({
+                            "type": "text",
+                            "ts": roman_cache_start / 1000,
+                            "end_ts": roman_cache_end / 1000,
+                            "value": roman_cache
+                        })
+                    roman_cache = ""
                     turn.append({
                         "type": "text",
                         "ts": (j.OffsetStartMs + start) / 1000,
                         "end_ts": (j.OffsetEndMs + start) / 1000,
                         "value": word
                     })
-                turns.append({
-                    "elements": turn,
-                    "speaker": i.SpeakerId
+
+            if roman_cache != "":
+                turn.append({
+                    "type": "text",
+                    "ts": roman_cache_start / 1000,
+                    "end_ts": roman_cache_end / 1000,
+                    "value": roman_cache
                 })
-            L.debug(f"Tencent done.")
-            
-            # Extract the text from the small volume parts for translation
-
-            doc = process_generation({"monologues": turns},
-                                    self.__lang_code, 
-                                    utterance_engine=self.__engine)
-            media = Media(type=MediaType.AUDIO, name=Path(f).stem, url=f)
-            doc.media = media
-            return doc
-            
-        finally:
-            if processed_path != f and pathlib.Path(processed_path).exists():
-                pathlib.Path(processed_path).unlink()
+
+            turns.append({
+                "elements": turn,
+                "speaker": i.SpeakerId
+            })
+        L.debug(f"Tencent done.")
+
+        # Extract the text from the small volume parts for translation
+
+        doc = process_generation({"monologues": turns},
+                                self.__lang_code, 
+                                utterance_engine=self.__engine)
+        media = Media(type=MediaType.AUDIO, name=Path(f).stem, url=f)
+        doc.media = media
+        return doc
+

{batchalignhk-0.7.19.post9 → batchalignhk-0.7.19.post11}/batchalign/pipelines/cleanup/retrace.py

@@ -22,7 +22,7 @@ class NgramRetraceEngine(BatchalignEngine):
                 if i.type in [TokenType.REGULAR, TokenType.PUNCT, TokenType.FP]:
                     content.append(i)
             # scan for n-gram retraces
-            for n in range(1, len(content)):
+            for n in range(1 if "yue" not in doc.langs and "zho" not in doc.langs else 2, len(content)):
                 begin = 0
                 while begin < len(content)-(n):
                     # get the n gram info; we convert it to

batchalignhk-0.7.19.post11/batchalign/version

@@ -0,0 +1,3 @@
+0.7.19-post.11
+May 27th, 2025
+tencent: tokenize roman words together

batchalignhk-0.7.19.post9/batchalign/version

@@ -1,3 +0,0 @@
-0.7.19-post.9
-May 24th, 2025
-reverts file only prep changes