batchalign 0.7.6a33__tar.gz → 0.7.7a2__tar.gz

{batchalign-0.7.6a33/batchalign.egg-info → batchalign-0.7.7a2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: batchalign
-Version: 0.7.6a33
+Version: 0.7.7a2
 Summary: Python Speech Language Sample Analysis
 Author: Brian MacWhinney, Houjun Liu
 Author-email: macw@cmu.edu, houjun@cmu.edu

{batchalign-0.7.6a33 → batchalign-0.7.7a2}/batchalign/formats/chat/utils.py

@@ -108,7 +108,7 @@ def annotation_clean(content, special=False):
     cleaned_word = re.sub(r"\x15\d+_\d+\x15", '', cleaned_word)
     if not special:
         cleaned_word = re.sub(r"&~\w+", '', cleaned_word)
-    cleaned_word = cleaned_word.replace("(","").replace(")","")
+    # cleaned_word = cleaned_word.replace("(","").replace(")","")
     cleaned_word = cleaned_word.replace("[","").replace("]","")
     cleaned_word = cleaned_word.replace("<","").replace(">","")
     cleaned_word = cleaned_word.replace("“","").replace("”","")

{batchalign-0.7.6a33 → batchalign-0.7.7a2}/batchalign/pipelines/analysis/eval.py

@@ -3,6 +3,7 @@ eval.py
 Engines for transcript evaluation
 """
 
+import re
 from batchalign.document import *
 from batchalign.pipelines.base import *
 from batchalign.pipelines.asr.utils import *
@@ -22,11 +23,34 @@ class EvaluationEngine(BatchalignEngine):
         forms = [ j.text.lower() for i in doc.content for j in i.content if isinstance(i, Utterance)]
         gold_forms = [ j.text.lower() for i in gold.content for j in i.content if isinstance(i, Utterance)]
 
-        forms = [i for i in forms if i.strip() not in MOR_PUNCT+ENDING_PUNCT]
-        gold_forms = [i for i in gold_forms if i.strip() not in MOR_PUNCT+ENDING_PUNCT]
+        forms = [i.replace("-", "") for i in forms if i.strip() not in MOR_PUNCT+ENDING_PUNCT]
+        gold_forms = [i.replace("-", "") for i in gold_forms if i.strip() not in MOR_PUNCT+ENDING_PUNCT]
+
+        forms = [re.sub(r"\((.*)\)",r"", i) for i in forms]
+        gold_forms = [re.sub(r"\((.*)\)",r"", i) for i in gold_forms]
+
+        # if there are single letter frames, we combine them tofgether
+        # until the utterance is done or there isn't any left
+        forms_finished = []
+
+        single_sticky = ""
+        is_single = False
+
+        for i in forms:
+            if len(i) == 1:
+                single_sticky += i
+            else:
+                if single_sticky != "":
+                    forms_finished.append(single_sticky)
+                    single_sticky = ""
+                forms_finished.append(i)
+
+        if single_sticky != "":
+            forms_finished.append(single_sticky)
+            single_sticky = ""
 
         # dp!
-        alignment = align(forms, gold_forms, False)
+        alignment = align(forms_finished, gold_forms, False)
 
         # calculate each type of error
         sub = 0
@@ -39,14 +63,28 @@ class EvaluationEngine(BatchalignEngine):
         #     but if we have <extra.reference> <extra.reference> this is 2 insertions
 
         cleaned_alignment = []
+        # whether we had a "firstname" in reference document and hence are
+        # anticipating a payload for it (the actual name) in the next entry in the
+        # alignment
+        anticipating_payload = False
 
         for i in alignment:
 
             if isinstance(i, Extra):
-                if len(cleaned_alignment) > 0 and i.extra_type == ExtraType.REFERENCE and "name" in i.key and i.key[:4] != "name":
-                    cleaned_alignment.pop(-1)
+
+                if i.extra_type == ExtraType.REFERENCE and "name" in i.key and i.key[:4] != "name":
+                    if (isinstance(cleaned_alignment[-1], Extra) and
+                        cleaned_alignment[-1].extra_type ==  ExtraType.PAYLOAD and
+                        len(cleaned_alignment) > 0):
+                        cleaned_alignment.pop(-1)
+                    else:
+                        anticipating_payload = True
                     cleaned_alignment.append(Match(i.key, None, None))
                     continue
+                elif i.extra_type == ExtraType.PAYLOAD and anticipating_payload:
+                    anticipating_payload = False
+                    continue
+            
 
                 if prev_error != None and prev_error != i.extra_type:
                     # this is a substitution: we have different "extra"s in
@@ -75,7 +113,7 @@ class EvaluationEngine(BatchalignEngine):
             cleaned_alignment.append(i)
 
         diff = []
-        for i in alignment:
+        for i in cleaned_alignment:
             if isinstance(i, Extra):
                 diff.append(f"{'+' if i.extra_type == ExtraType.REFERENCE else '-'} {i.key}")
             else:

{batchalign-0.7.6a33 → batchalign-0.7.7a2}/batchalign/pipelines/morphosyntax/ud.py

@@ -115,6 +115,7 @@ def handler(word, lang=None):
     target = target.replace('/100', '')
     target = target.replace('/r', '')
     target = target.replace('(', '')
+    target = target.replace("(","").replace(")","")
 
     # remove attachments
     if "|" in target:
@@ -848,7 +849,7 @@ def morphoanalyze(doc: Document, retokenize:bool, status_hook:callable = None, *
         inputs.append(line_cut)
 
         try:
-            sents = nlp(line_cut.strip()).sentences
+            sents = nlp(line_cut.replace("(","").replace(")","").strip()).sentences
 
             if len(sents) == 0:
                 continue

{batchalign-0.7.6a33 → batchalign-0.7.7a2}/batchalign/pipelines/utterance/ud_utterance.py

@@ -84,6 +84,7 @@ def parse_tree(subtree):
                      for i in stack]
 
 def process_ut(ut, nlp):
+
     # remove punct
     if (ut.content[-1].type == TokenType.PUNCT or
         ut.content[-1].text in ENDING_PUNCT):
@@ -142,7 +143,7 @@ def process_ut(ut, nlp):
         if isinstance(i, Match):
             matches.append(i)
         elif i.extra_type == ExtraType.REFERENCE:
-            new_refs.append(ReferenceTarget(key=i.key, payload=i.payload))
+            new_refs.append(ReferenceTarget(key=i.key, payload=i.payload if i.payload else -1))
 
     # we now sort the references based on their orignial utterance order
     matches = matches + new_refs

{batchalign-0.7.6a33 → batchalign-0.7.7a2}/batchalign/utils/utils.py

@@ -29,6 +29,7 @@ def word_tokenize(str):
         return tmp.tokenize(str)
     except LookupError:
         nltk.download("punkt")
+        nltk.download("punkt_tab")
         return tmp.tokenize(str)
 
 def sent_tokenize(str):
@@ -49,6 +50,7 @@ def sent_tokenize(str):
         return ST(str)
     except LookupError:
         nltk.download("punkt")
+        nltk.download("punkt_tab")
         return ST(str)
 
 def detokenize(tokens):
@@ -69,6 +71,7 @@ def detokenize(tokens):
         return TreebankWordDetokenizer().detokenize(tokens)
     except LookupError:
         nltk.download("punkt")
+        nltk.download("punkt_tab")
         return TreebankWordDetokenizer().detokenize(tokens)
 
 def correct_timing(doc):

batchalign-0.7.7a2/batchalign/version

@@ -0,0 +1,3 @@
+0.7.7-alpha.2
+December 06, 2024
+DP logic

{batchalign-0.7.6a33 → batchalign-0.7.7a2/batchalign.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: batchalign
-Version: 0.7.6a33
+Version: 0.7.7a2
 Summary: Python Speech Language Sample Analysis
 Author: Brian MacWhinney, Houjun Liu
 Author-email: macw@cmu.edu, houjun@cmu.edu

batchalign-0.7.6a33/batchalign/version

@@ -1,3 +0,0 @@
-0.7.6-alpha.33
-November 26, 2024
-French APM (minor)