BatchalignHK 0.7.20.post3__tar.gz → 0.7.20.post5__tar.gz

{batchalignhk-0.7.20.post3 → batchalignhk-0.7.20.post5}/BatchalignHK.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.2
 Name: BatchalignHK
-Version: 0.7.20.post3
+Version: 0.7.20.post5
 Summary: Python Speech Language Sample Analysis
 Author: Brian MacWhinney, Houjun Liu
 Author-email: macw@cmu.edu, houjun@cmu.edu
@@ -16,13 +16,14 @@ Requires-Dist: torchaudio
 Requires-Dist: opencc-python-reimplemented
 Requires-Dist: pydub
 Requires-Dist: plotly>=5.3.0
-Requires-Dist: transformers>=4.37
+Requires-Dist: transformers>=4.38.2
 Requires-Dist: tokenizers>=0.14.1
 Requires-Dist: pycountry>=22.3
 Requires-Dist: stanza[transformers]>=1.10.1
 Requires-Dist: scipy~=1.11
 Requires-Dist: rev_ai>=2.18.0
 Requires-Dist: rich~=13.6
+Requires-Dist: omegaconf
 Requires-Dist: click~=8.1
 Requires-Dist: matplotlib<4.0.0,>=3.8.0
 Requires-Dist: pyfiglet==1.0.2
@@ -52,7 +53,6 @@ Dynamic: author-email
 Dynamic: classifier
 Dynamic: description
 Dynamic: description-content-type
-Dynamic: license-file
 Dynamic: provides-extra
 Dynamic: requires-dist
 Dynamic: summary

{batchalignhk-0.7.20.post3 → batchalignhk-0.7.20.post5}/BatchalignHK.egg-info/requires.txt

@@ -6,13 +6,14 @@ torchaudio
 opencc-python-reimplemented
 pydub
 plotly>=5.3.0
-transformers>=4.37
+transformers>=4.38.2
 tokenizers>=0.14.1
 pycountry>=22.3
 stanza[transformers]>=1.10.1
 scipy~=1.11
 rev_ai>=2.18.0
 rich~=13.6
+omegaconf
 click~=8.1
 matplotlib<4.0.0,>=3.8.0
 pyfiglet==1.0.2

{batchalignhk-0.7.20.post3 → batchalignhk-0.7.20.post5}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.2
 Name: BatchalignHK
-Version: 0.7.20.post3
+Version: 0.7.20.post5
 Summary: Python Speech Language Sample Analysis
 Author: Brian MacWhinney, Houjun Liu
 Author-email: macw@cmu.edu, houjun@cmu.edu
@@ -16,13 +16,14 @@ Requires-Dist: torchaudio
 Requires-Dist: opencc-python-reimplemented
 Requires-Dist: pydub
 Requires-Dist: plotly>=5.3.0
-Requires-Dist: transformers>=4.37
+Requires-Dist: transformers>=4.38.2
 Requires-Dist: tokenizers>=0.14.1
 Requires-Dist: pycountry>=22.3
 Requires-Dist: stanza[transformers]>=1.10.1
 Requires-Dist: scipy~=1.11
 Requires-Dist: rev_ai>=2.18.0
 Requires-Dist: rich~=13.6
+Requires-Dist: omegaconf
 Requires-Dist: click~=8.1
 Requires-Dist: matplotlib<4.0.0,>=3.8.0
 Requires-Dist: pyfiglet==1.0.2
@@ -52,7 +53,6 @@ Dynamic: author-email
 Dynamic: classifier
 Dynamic: description
 Dynamic: description-content-type
-Dynamic: license-file
 Dynamic: provides-extra
 Dynamic: requires-dist
 Dynamic: summary

{batchalignhk-0.7.20.post3 → batchalignhk-0.7.20.post5}/batchalign/formats/chat/lexer.py

@@ -47,7 +47,7 @@ class UtteranceLexer:
                     decoded.append((i, type))
         return decoded
 
-    def __get_until(self, end_tokens=[' ']):
+    def _get_until(self, end_tokens=[' ']):
         has_read_nonempty = False
         tokens = []
         while True:
@@ -79,7 +79,8 @@ class UtteranceLexer:
             # self.__clauses.append((form.strip(), TokenType.FEAT))
         elif form.strip() in NORMAL_GROUP_MARKS:
             # basically ignore the form
-            pass
+            self.__clauses.append((self.__clauses.pop(-1)[0], TokenType.REGULAR))
+            # pass
             # self.__clauses.append((form.strip(), TokenType.FEAT))
         elif form[0] == "[" and form[:2] != "[:":
             # we ignore all other things which are simple annotations
@@ -97,13 +98,13 @@ class UtteranceLexer:
             self.__clauses.append((annotation_clean(form).strip(), TokenType.REGULAR))
 
     def __pull(self):
-        form, num, delim = self.__get_until()
+        form, num, delim = self._get_until()
 
         self.__handle(form, num, delim)
 
         return form
 
-    def __get_group(self, form, type):
+    def _get_group(self, form, type):
         text = ""
         group = [form]
 
@@ -113,11 +114,10 @@ class UtteranceLexer:
 
         # scan forward until we have the first actual form, if
         # its a selection group
-        if type == ">" and annotation_clean(form, special=True) == "":
-            form, num, delim = self.__get_until()
+        if type == ">" and annotation_clean(form, special=True) == "" and form != "<":
+            form, num, delim = self._get_until()
             group = [group.pop(0).strip()+annotation_clean(form)]
 
-
         # decrement nesting first
         if form not in REPEAT_GROUP_MARKS and form not in NORMAL_GROUP_MARKS:
             if type == ">" and ">" in form:
@@ -127,7 +127,7 @@ class UtteranceLexer:
 
         # grab forward the entire group 
         while (type not in form) or (nesting != -1):
-            form, num, delim = self.__get_until()
+            form, num, delim = self._get_until()
 
             sform = copy.deepcopy(form)
             for i in REPEAT_GROUP_MARKS + NORMAL_GROUP_MARKS:
@@ -157,7 +157,6 @@ class UtteranceLexer:
         words = [re.compile(r"[^A-Za-zÀ-ÖØ-öø-ÿ']").sub("", i).strip() for i in group
                  if re.compile(r"[^A-Za-zÀ-ÖØ-öø-ÿ']").sub("", i).strip()!= ""]
 
-
         if type == "]":
             return words, special[0], text
         else:
@@ -167,7 +166,7 @@ class UtteranceLexer:
         orig_form = form
 
         # scan the group
-        words, special, text = self.__get_group(form, type)
+        words, special, text = self._get_group(form, type)
         text = form + text
 
         if len(text.strip()) == 0:

batchalignhk-0.7.20.post5/batchalign/version

@@ -0,0 +1,3 @@
+0.7.20-post.5
+July 25th, 2025
+Patch lexer fixes (wip?)

{batchalignhk-0.7.20.post3 → batchalignhk-0.7.20.post5}/setup.py

@@ -38,13 +38,14 @@ setup(
         "pydub",
         # "imblearn",
         "plotly>=5.3.0",
-        "transformers>=4.37",
+        "transformers>=4.38.2",
         "tokenizers>=0.14.1",
         "pycountry>=22.3",
         "stanza[transformers]>=1.10.1",
         "scipy~=1.11",
         "rev_ai>=2.18.0",
         "rich~=13.6",
+        "omegaconf",
         "click~=8.1",
         "matplotlib>=3.8.0,<4.0.0",
         "pyfiglet==1.0.2",

batchalignhk-0.7.20.post3/batchalign/version

@@ -1,3 +0,0 @@
-0.7.20-post.3
-July 21th, 2025
-Some benchmarking abbreviations