PyPI - batchalign - Versions diffs - 0.7.1b13__tar.gz → 0.7.1b14__tar.gz - Mend

@@ -130,7 +130,8 @@ def handler(word, lang=None):
     # fix dash
     target = target.replace("-", "–")
-    if target == "“":
+    if "“" in target:
         target = word.text
     return f"{'' if not unknown else '0'}{word.upos.lower()}|{target}"
@@ -741,7 +742,6 @@ def morphoanalyze(doc: Document, retokenize:bool, status_hook:callable = None, *
             line_cut = i.strip(join_with_spaces=True)
             ending = '.'
         # clean the sentence
         line_cut = clean_sentence(line_cut)
@@ -818,12 +818,12 @@ def morphoanalyze(doc: Document, retokenize:bool, status_hook:callable = None, *
                 chunks_align = [(i,j) for i,j in chunks
                                 if len(j) != 0 and (j[0] not in ["<", "[", "&", "\x15", "(", ")"]) and (j[-1] not in ["]"])
                                                    and ("@" not in j)
-                                and j.strip() not in ENDING_PUNCT + MOR_PUNCT + CHAT_IGNORE + ["++"]]
+                                and j.strip() not in MOR_PUNCT + CHAT_IGNORE + ["++"]]
                 # hollow out anything we are trying to align, and leave everything else
                 chunks_backplate = [[j]
                                     if not (len(j) != 0 and (j[0] not in ["<", "[", "&", "\x15", "(", ")"]) and (j[-1] not in ["]"])
                                     and ("@" not in j)
-                                            and j.strip() not in ENDING_PUNCT + MOR_PUNCT + CHAT_IGNORE + ["++"])
+                                            and j.strip() not in MOR_PUNCT + CHAT_IGNORE + ["++"])
                                     else
                                     []
                                     for i,j in chunks]
@@ -845,6 +845,14 @@ def morphoanalyze(doc: Document, retokenize:bool, status_hook:callable = None, *
                     elif isinstance(i, Extra) and i.extra_type == ExtraType.PAYLOAD:
                         # just put it back
                         chunks_backplate[i.payload].append(i.key)
+                    # we want to replace the morphology of forms that are not actually
+                    # supposed to be analyzed
+                    elif isinstance(i, Extra) and i.extra_type == ExtraType.REFERENCE:
+                        ut[i.payload].morphology = [Morphology(
+                            lemma = sents[0].tokens[i.payload].text,
+                            pos = "x",
+                            feats = ""
+                        )]
                 # resolve all the numbers and flatten
                 chunks_backplate = [j if isinstance(j, str) else ut[j].text
                                     for i in chunks_backplate
@@ -858,9 +866,9 @@ def morphoanalyze(doc: Document, retokenize:bool, status_hook:callable = None, *
                 retokenized_ut = retokenized_ut.replace("< ", "<")
                 retokenized_ut = retokenized_ut.replace(" :", ":")
                 retokenized_ut = retokenized_ut.replace(": <", ": <")
+                retokenized_ut = retokenized_ut.replace(" ↑", "↑")
                 retokenized_ut = re.sub(r"@ ?w ?p", "@wp", retokenized_ut)
                 retokenized_ut = retokenized_ut.replace(" @", "@")
-                retokenized_ut = re.sub(r" ↑", "↑", retokenized_ut)
                 # pray to everyone that it works---this will simply crash and ignore
                 # the utterance if it didn't work, so we are doing this as a sanity
                 # check rather than needing the parsed result

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: batchalign
-Version: 0.7.1b13
+Version: 0.7.1b14
 Summary: Python Speech Language Sample Analysis
 Author: Brian MacWhinney, Houjun Liu
 Author-email: macw@cmu.edu, houjun@cmu.edu

@@ -83,7 +83,6 @@ class TokenType(IntEnum):
     ANNOT = 4 # &~ject &~head
     PUNCT = 5 # ‡„,
     CORRECTION = 6 # test [= test]
-    VOCAL = 7 # test@o
 class CustomLineType(IntEnum):
     DEPENDENT = 0 # %com

@@ -93,8 +93,6 @@ class UtteranceLexer:
             self.__clauses.append((form, TokenType.FEAT))
         elif annotation_clean(form).strip() in CHAT_IGNORE:
             self.__clauses.append((annotation_clean(form).strip(), TokenType.ANNOT))
-        elif "@" in form:
-            self.__clauses.append((annotation_clean(form).strip(), TokenType.VOCAL))
         else:
             self.__clauses.append((annotation_clean(form).strip(), TokenType.REGULAR))

@@ -90,10 +90,9 @@ def chat_parse_utterance(text, mor, gra, wor, additional):
     # seperate out main words by whether it should have phonation/morphology and add ending punct
     words = list(enumerate(tokens))
     lexed_words = [tok for tok in words if tok[1][1] in [TokenType.REGULAR,
-                                                         TokenType.PUNCT]]
+                                                            TokenType.PUNCT]]
     phonated_words = [tok for tok in words if tok[1][1] in [TokenType.REGULAR,
                                                             TokenType.RETRACE,
-                                                            TokenType.VOCAL,
                                                             TokenType.PUNCT,
                                                             TokenType.FP]]
     # create base forms

@@ -0,0 +1,3 @@
+0.7.1-beta.14
+June 02nd, 2024
+more UD bugs, superceeds v0.7.1-beta.13

@@ -1,3 +0,0 @@
-0.7.1-beta.13
-June 2nd, 2024
-more ud fixes

batchalign 0.7.1b13__tar.gz → 0.7.1b14__tar.gz

batchalign 0.7.1b13tar.gz → 0.7.1b14tar.gz