BatchalignHK 0.7.22.post9__tar.gz → 0.7.22.post11__tar.gz

{batchalignhk-0.7.22.post9 → batchalignhk-0.7.22.post11}/BatchalignHK.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: BatchalignHK
-Version: 0.7.22.post9
+Version: 0.7.22.post11
 Summary: Python Speech Language Sample Analysis
 Author: Brian MacWhinney, Houjun Liu
 Author-email: macw@cmu.edu, houjun@cmu.edu

{batchalignhk-0.7.22.post9 → batchalignhk-0.7.22.post11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: BatchalignHK
-Version: 0.7.22.post9
+Version: 0.7.22.post11
 Summary: Python Speech Language Sample Analysis
 Author: Brian MacWhinney, Houjun Liu
 Author-email: macw@cmu.edu, houjun@cmu.edu

{batchalignhk-0.7.22.post9 → batchalignhk-0.7.22.post11}/batchalign/pipelines/analysis/eval.py

@@ -18,134 +18,136 @@ import logging
 L = logging.getLogger("batchalign")
 
 joined_compounds = ["".join(k) for k in compounds]
-lowered_abbrev = [k.lower() for k in abbrev]
+lowered_abbrev = [k for k in abbrev]
 
 fillers = ["um", "uhm", "em", "mhm", "uhhm", "eh", "uh", "hm"]
 def conform(x):
     result = []
     for i in x:
-        if i.strip() in joined_compounds:
+        if i.strip().lower() in joined_compounds:
             for k in compounds[joined_compounds.index(i.strip())]:
                 result.append(k)
         elif i.strip() in lowered_abbrev:
             for j in i.strip():
-                result.append(j)
-        elif "'s" in i.strip():
+                result.append(j.strip())
+        elif "'s" in i.strip().lower():
             result.append(i.split("'")[0])
             result.append("is")
-        elif "'ve" in i.strip():
+        elif "'ve" in i.strip().lower():
             result.append(i.split("'")[0])
             result.append("have")
-        elif "'s" in i.strip():
+        elif "'s" in i.strip().lower():
             result.append(i.split("'")[0])
             result.append("is")
-        elif "'d" in i.strip():
+        elif "'d" in i.strip().lower():
             result.append(i.split("'")[0])
             result.append("had")
-        elif "'m" in i.strip():
+        elif "'m" in i.strip().lower():
             result.append(i.split("'")[0])
             result.append("am")
-        elif i.strip() in fillers:
+        elif i.strip().lower() in fillers:
             result.append("um")
-        elif "-" in i.strip():
-            result += [k.strip() for k in i.split("-")]
-        elif "ok" == i.strip():
+        elif "-" in i.strip().lower():
+            result += [k.strip() for k in i.split("-").lower()]
+        elif "ok" == i.strip().lower():
             result.append("okay")
-        elif "gimme" == i.strip():
+        elif "gimme" == i.strip().lower():
             result.append("give")
             result.append("me")
-        elif "hafta" == i.strip() or "havta" == i.strip():
+        elif "hafta" == i.strip().lower() or "havta" == i.strip().lower():
             result.append("have")
             result.append("to")
-        elif i.strip() in names:
+        elif i.strip().lower() in names:
             result.append("name")
-        elif "dunno" == i.strip():
+        elif "dunno" == i.strip().lower():
             result.append("don't")
             result.append("know")
-        elif "wanna" == i.strip():
+        elif "wanna" == i.strip().lower():
             result.append("want")
             result.append("to")
-        elif "mba" == i.strip():
+        elif "mba" == i.strip().lower():
             result.append("m")
             result.append("b")
             result.append("a")
-        elif "tli" == i.strip():
+        elif "tli" == i.strip().lower():
             result.append("t")
             result.append("l")
             result.append("i")
-        elif "bbc" == i.strip():
+        elif "bbc" == i.strip().lower():
             result.append("b")
             result.append("b")
             result.append("c")
-        elif "ii" == i.strip():
+        elif "ii" == i.strip().lower():
             result.append("i")
             result.append("i")
-        elif "i'd" == i.strip():
+        elif "i'd" == i.strip().lower():
             result.append("i")
             result.append("had")
-        elif "alright" == i.strip():
+        elif "alright" == i.strip().lower():
             result.append("all")
             result.append("right")
-        elif "sorta" == i.strip():
+        elif "sorta" == i.strip().lower():
             result.append("sort")
             result.append("of")
-        elif "alrightie" == i.strip():
+        elif "alrightie" == i.strip().lower():
             result.append("all")
             result.append("right")
-        elif "mm" == i.strip():
+        elif "mm" == i.strip().lower():
             result.append("hm")
-        elif "ai" == i.strip():
+        elif "ai" == i.strip().lower():
             result.append("a")
             result.append("i")
-        elif "this'll" == i.strip():
+        elif "this'll" == i.strip().lower():
             result.append("this")
             result.append("will")
-        elif "gotta" == i.strip():
+        elif "gotta" == i.strip().lower():
             result.append("got")
             result.append("to")
-        elif "hadta" == i.strip():
+        elif "hadta" == i.strip().lower():
             result.append("had")
             result.append("to")
-        elif "eh" == i.strip():
+        elif "eh" == i.strip().lower():
             result.append("uh")
-        elif "kinda" == i.strip():
+        elif "kinda" == i.strip().lower():
             result.append("kind")
             result.append("of")
-        elif "ed" == i.strip():
+        elif "ed" == i.strip().lower():
             result.append("education")
-        elif "til" == i.strip():
+        elif "til" == i.strip().lower():
             result.append("until")
-        elif "gonna" == i.strip():
+        elif "gonna" == i.strip().lower():
             result.append("going")
             result.append("to")
-        elif "shoulda" == i.strip():
+        elif "shoulda" == i.strip().lower():
             result.append("should")
             result.append("have")
-        elif "sposta" == i.strip():
+        elif "sposta" == i.strip().lower():
             result.append("supposed")
             result.append("to")
-        elif "farmhouse" == i.strip():
+        elif "farmhouse" == i.strip().lower():
             result.append("farm")
             result.append("house")
-        elif "aa" == i.strip():
+        elif "aa" == i.strip().lower():
             result.append("a")
             result.append("a")
-        elif "aa" == i.strip():
+        elif "aa" == i.strip().lower():
             result.append("a")
             result.append("a")
-        elif "em" == i.strip():
+        elif "em" == i.strip().lower():
             result.append("them")
-        elif "hmm" == i.strip():
+        elif "hmm" == i.strip().lower():
             result.append("hm")
-        elif "_" in i.strip():
+        elif "_" in i.strip().lower():
             for j in i.strip().split("_"):
                 result.append(j)
         else:
-            result.append(i)
+            result.append(i.lower())
 
     return result
 
 def match_fn(x,y):
+    x = x.lower()
+    y = y.lower()
     return (y == x or
             y.replace("(", "").replace(")", "") == x.replace("(", "").replace(")", "") or
             re.sub(r"\((.*)\)",r"", y) == x or re.sub(r"\((.*)\)",r"", x) == y)
@@ -156,8 +158,8 @@ class EvaluationEngine(BatchalignEngine):
     @staticmethod
     def __compute_wer(doc, gold):
         # get the text of the document and get the text of the gold
-        forms = [ j.text.lower() for i in doc.content for j in i.content if isinstance(i, Utterance)]
-        gold_forms = [ j.text.lower() for i in gold.content for j in i.content if isinstance(i, Utterance)]
+        forms = [ j.text for i in doc.content for j in i.content if isinstance(i, Utterance)]
+        gold_forms = [ j.text for i in gold.content for j in i.content if isinstance(i, Utterance)]
 
         forms = [i.replace("-", "") for i in forms if i.strip() not in MOR_PUNCT+ENDING_PUNCT]
         gold_forms = [i.replace("-", "") for i in gold_forms if i.strip() not in MOR_PUNCT+ENDING_PUNCT]

{batchalignhk-0.7.22.post9 → batchalignhk-0.7.22.post11}/batchalign/pipelines/asr/rev.py

@@ -86,6 +86,7 @@ class RevEngine(BatchalignEngine):
         while status == JobStatus.IN_PROGRESS:
             time.sleep(15)
             status = client.get_job_details(job.id).status
+            L.debug(f"Rev.AI got '{status}'...")
 
         # if we failed, report failure and give up
         if status == JobStatus.FAILED:

{batchalignhk-0.7.22.post9 → batchalignhk-0.7.22.post11}/batchalign/pipelines/asr/utils.py

@@ -60,10 +60,11 @@ def retokenize(intermediate_output):
             word = word.replace("。", ".")
             word = word.replace("¿", " ").replace("¡", " ")
             tmp.append((word, bullet))
-            if len(word) > 0 and (word in ENDING_PUNCT or word[-1] in ENDING_PUNCT):
-                if word in ENDING_PUNCT:
+            if len(word) > 0 and (word in ENDING_PUNCT+["؟", "۔", "،", "؛"]
+                                  or word[-1] in ENDING_PUNCT+["؟", "۔", "،", "؛"]):
+                if word in ENDING_PUNCT+["؟", "۔", "،", "؛"]:
                     final_outputs.append((speaker, tmp))
-                elif word[-1] in ENDING_PUNCT:
+                elif word[-1] in ENDING_PUNCT+["؟", "۔", "،", "؛"]:
                     # we want to seperate the ending punct out
                     final, time = tmp.pop(-1)
                     tmp.append((final[:-1], time))
@@ -102,7 +103,7 @@ def retokenize_with_engine(intermediate_output, engine):
         # because we are using an utterance engine, we need
         # to get rid of all the preexisting punctuation
         for i in utterance:
-            for j in MOR_PUNCT+ENDING_PUNCT:
+            for j in MOR_PUNCT+ENDING_PUNCT+["؟", "۔", "،", "؛"]:
                 i[0] = i[0].strip(j).lower()
 
         # remove everything that's now blank
@@ -118,7 +119,7 @@ def retokenize_with_engine(intermediate_output, engine):
         # align the utterance against original splits and generate final outputs
         for i in split:
             # Check if the split has ending punctuation
-            if i[-1] in ENDING_PUNCT:
+            if i[-1] in ENDING_PUNCT+["؟", "۔", "،", "؛"]:
                 new_ut, delim = (i[:-1].split(" "), i[-1])
             else:
                 new_ut, delim = (i.split(" "), ".")
@@ -273,16 +274,8 @@ def process_generation(output, lang="eng", utterance_engine=None):
                 seen_word = False
             if word.strip() == "":
                 continue
-            if word not in ENDING_PUNCT+MOR_PUNCT:
+            if word not in ENDING_PUNCT+MOR_PUNCT+["؟", "۔", "،", "؛"]:
                 word_replaced = word
-                if word_replaced.strip() == "؟":
-                    word_replaced = "?"
-                elif word_replaced.strip() == "۔":
-                    word_replaced = "."
-                elif word_replaced.strip() == "،":
-                    word_replaced = ","
-                elif word_replaced.strip() == "؛":
-                    word_replaced = ";"
                     
                 if start == None or end == None:
                     words.append(Form(text=word_replaced, time=None))
@@ -290,7 +283,15 @@ def process_generation(output, lang="eng", utterance_engine=None):
                     seen_word = True
                     words.append(Form(text=word_replaced, time=(int(start), int(end))))
             else:
-                    words.append(Form(text=word, time=None))
+                if word.strip() == "؟":
+                    word = "?"
+                elif word.strip() == "۔":
+                    word = "."
+                elif word.strip() == "،":
+                    word = ","
+                elif word.strip() == "؛":
+                    word = ";"
+                words.append(Form(text=word, time=None))
 
         final_utterances.append(Utterance(
             tier=participant,

batchalignhk-0.7.22.post11/batchalign/version

@@ -0,0 +1,3 @@
+0.7.22-post.11
+October 2st, 2025
+Fix arabic punctuation

batchalignhk-0.7.22.post9/batchalign/version

@@ -1,3 +0,0 @@
-0.7.22-post.9
-September 29th, 2025
-Arabic Commas