batchalign 0.7.6a12__tar.gz → 0.7.6a14__tar.gz

{batchalign-0.7.6a12/batchalign.egg-info → batchalign-0.7.6a14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: batchalign
-Version: 0.7.6a12
+Version: 0.7.6a14
 Summary: Python Speech Language Sample Analysis
 Author: Brian MacWhinney, Houjun Liu
 Author-email: macw@cmu.edu, houjun@cmu.edu

{batchalign-0.7.6a12 → batchalign-0.7.6a14}/batchalign/document.py

@@ -324,8 +324,7 @@ class Utterance(BaseModel):
             to_include.append(TokenType.RETRACE)
         if include_fp:
             to_include.append(TokenType.FP)
-        filtered = filter(lambda x:x.type in to_include,
-                          self.content)
+        filtered = filter(lambda x:x.type in to_include, self.content)
         # chain them together
         if join_with_spaces:
             return " ".join([i.text for i in filtered])

{batchalign-0.7.6a12 → batchalign-0.7.6a14}/batchalign/formats/chat/utils.py

@@ -146,7 +146,7 @@ def annotation_clean(content, special=False):
     cleaned_word = cleaned_word.replace("~","").replace("&~","")
     cleaned_word = cleaned_word.replace(">","").replace("<","")
     cleaned_word = cleaned_word.replace("〕","").replace("//","").replace(";","")
-    cleaned_word = re.sub(r"@[^abcefpoqs]", '', cleaned_word)
+    cleaned_word = re.sub(r"@[^abcefpoqsw]", '', cleaned_word)
     cleaned_word = re.sub(r"&.", '', cleaned_word)
 
     return cleaned_word

{batchalign-0.7.6a12 → batchalign-0.7.6a14}/batchalign/pipelines/morphosyntax/ud.py

@@ -837,11 +837,17 @@ def morphoanalyze(doc: Document, retokenize:bool, status_hook:callable = None, *
                 L.debug(f"Encountered an utterance that's likely devoid of morphological information; skipping... utterance='{doc.content[indx]}'")
                 continue
 
+
             if retokenize:
                 # rewrite the sentence with our desired tokenizations
                 ut, end = chat_parse_utterance(" ".join([i.text for i in sents[0].tokens])+" "+ending,
                                           mor, gra,
                                           None, None)
+                # fix xbxxx
+                for i in ut:
+                    if i.text == "xbxxx" and len(i.morphology) > 0:
+                        i.text = i.morphology[0].lemma
+
                 # split the text up into previous chunks
                 chunks = list(enumerate(doc.content[indx].text.split(" ")))
                 # filter out everything that could not possibly align
@@ -866,6 +872,8 @@ def morphoanalyze(doc: Document, retokenize:bool, status_hook:callable = None, *
                 for i,j in enumerate(ut):
                     for k in j.text:
                         ud_chars.append(ReferenceTarget(k, payload=i))
+                creaky = False
+                collected = ""
                 # brrr
                 aligned = align(chunks_chars, ud_chars, tqdm=False)
                 for i in aligned:
@@ -873,16 +881,23 @@ def morphoanalyze(doc: Document, retokenize:bool, status_hook:callable = None, *
                         if i.reference_payload not in chunks_backplate[i.payload]:
                             chunks_backplate[i.payload].append(i.reference_payload)
                     elif isinstance(i, Extra) and i.extra_type == ExtraType.PAYLOAD:
-                        # just put it back
-                        chunks_backplate[i.payload].append(i.key)
+                        if i.key == "*":
+                            creaky = not creaky
+                            chunks_backplate[i.payload].append("*"+collected+"*")
+                            collected = ""
+                        elif creaky:
+                            collected += i.key
+                        elif not creaky:
+                            chunks_backplate[i.payload].append(i.key)
                     # we want to replace the morphology of forms that are not actually
                     # supposed to be analyzed
                     elif isinstance(i, Extra) and i.extra_type == ExtraType.REFERENCE:
-                        ut[i.payload].morphology = [Morphology(
-                            lemma = sents[0].tokens[i.payload].text,
-                            pos = "x",
-                            feats = ""
-                        )]
+                        if ut[i.payload].text != ",":
+                            ut[i.payload].morphology = [Morphology(
+                                lemma = sents[0].tokens[i.payload].text if len(sents) > 0 and len(sents[0].tokens) > i.payload and sents[0].tokens[i.payload].text != "xbxxx" else ut[i.payload].text,
+                                pos = "x",
+                                feats = ""
+                            )]
 
                 poses = [i.morphology[0].pos.upper() for i in ut
                          if i.morphology
@@ -908,6 +923,8 @@ def morphoanalyze(doc: Document, retokenize:bool, status_hook:callable = None, *
                 retokenized_ut = retokenized_ut.replace(" ↑", "↑")
                 retokenized_ut = re.sub(r"@ ?w ?p", "@wp", retokenized_ut)
                 retokenized_ut = retokenized_ut.replace(" @", "@")
+                retokenized_ut = re.sub(r"\*[* ]*", "*", retokenized_ut)
+                retokenized_ut = re.sub(r"\*(.*?)\*", r"*\1* ", retokenized_ut)
                 # pray to everyone that it works---this will simply crash and ignore
                 # the utterance if it didn't work, so we are doing this as a sanity
                 # check rather than needing the parsed result

batchalign-0.7.6a14/batchalign/version

@@ -0,0 +1,3 @@
+0.7.6-alpha.14
+October 14, 2024
+creaky

{batchalign-0.7.6a12 → batchalign-0.7.6a14/batchalign.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: batchalign
-Version: 0.7.6a12
+Version: 0.7.6a14
 Summary: Python Speech Language Sample Analysis
 Author: Brian MacWhinney, Houjun Liu
 Author-email: macw@cmu.edu, houjun@cmu.edu

batchalign-0.7.6a12/batchalign/version

@@ -1,3 +0,0 @@
-0.7.6-alpha.12
-October 10, 2024
-patch bug regarding comma structure