batchalign 0.7.3b15__tar.gz → 0.7.3b17__tar.gz

{batchalign-0.7.3b15/batchalign.egg-info → batchalign-0.7.3b17}/PKG-INFO

@@ -1,16 +1,76 @@
 Metadata-Version: 2.1
 Name: batchalign
-Version: 0.7.3b15
+Version: 0.7.3b17
 Summary: Python Speech Language Sample Analysis
 Author: Brian MacWhinney, Houjun Liu
 Author-email: macw@cmu.edu, houjun@cmu.edu
 Classifier: Development Status :: 3 - Alpha
 Classifier: Topic :: Utilities
 Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pydantic>=2.4
+Requires-Dist: nltk>=3.8
+Requires-Dist: praatio<6.1.0,>=6.0.0
+Requires-Dist: torch<2.2.0,>=2.0.1
+Requires-Dist: torchaudio<2.2.0,>=2.1.0
+Requires-Dist: pyAudioAnalysis==0.3.14
+Requires-Dist: hmmlearn==0.3.0
+Requires-Dist: eyed3~=0.9.7
+Requires-Dist: pydub<0.26.0,>=0.25.1
+Requires-Dist: imblearn
+Requires-Dist: plotly>=5.18.0
+Requires-Dist: transformers~=4.37
+Requires-Dist: tokenizers>=0.14.1
+Requires-Dist: pycountry>=22.3
+Requires-Dist: stanza>=1.7
+Requires-Dist: scipy~=1.11
+Requires-Dist: rev_ai>=2.18.0
+Requires-Dist: rich~=13.6
+Requires-Dist: click~=8.1
+Requires-Dist: matplotlib<4.0.0,>=3.8.0
+Requires-Dist: pyfiglet==1.0.2
+Requires-Dist: soundfile~=0.12.0
+Requires-Dist: rich-click>=1.7.0
+Requires-Dist: typing-extensions
 Provides-Extra: dev
+Requires-Dist: pytest; extra == "dev"
 Provides-Extra: train
+Requires-Dist: accelerate~=0.27; extra == "train"
 Provides-Extra: speaker
-License-File: LICENSE
+Requires-Dist: nemo-toolkit~=1.21.0; extra == "speaker"
+Requires-Dist: omegaconf~=2.3.0; extra == "speaker"
+Requires-Dist: pydub~=0.25.0; extra == "speaker"
+Requires-Dist: braceexpand; extra == "speaker"
+Requires-Dist: editdistance; extra == "speaker"
+Requires-Dist: g2p_en; extra == "speaker"
+Requires-Dist: ipywidgets; extra == "speaker"
+Requires-Dist: jiwer; extra == "speaker"
+Requires-Dist: kaldi-python-io; extra == "speaker"
+Requires-Dist: kaldiio; extra == "speaker"
+Requires-Dist: lhotse>=1.20.0; extra == "speaker"
+Requires-Dist: librosa>=0.10.0; extra == "speaker"
+Requires-Dist: marshmallow; extra == "speaker"
+Requires-Dist: matplotlib; extra == "speaker"
+Requires-Dist: packaging; extra == "speaker"
+Requires-Dist: pyannote.core; extra == "speaker"
+Requires-Dist: pyannote.metrics; extra == "speaker"
+Requires-Dist: pydub; extra == "speaker"
+Requires-Dist: pyloudnorm; extra == "speaker"
+Requires-Dist: resampy; extra == "speaker"
+Requires-Dist: ruamel.yaml; extra == "speaker"
+Requires-Dist: scipy>=0.14; extra == "speaker"
+Requires-Dist: soundfile; extra == "speaker"
+Requires-Dist: sox; extra == "speaker"
+Requires-Dist: texterrors; extra == "speaker"
+Requires-Dist: hydra-core<=1.3.2,>1.3; extra == "speaker"
+Requires-Dist: omegaconf<=2.3; extra == "speaker"
+Requires-Dist: pytorch-lightning>=2.2.1; extra == "speaker"
+Requires-Dist: torchmetrics>=0.11.0; extra == "speaker"
+Requires-Dist: transformers>=4.36.0; extra == "speaker"
+Requires-Dist: wandb; extra == "speaker"
+Requires-Dist: webdataset>=0.2.86; extra == "speaker"
+Requires-Dist: sentencepiece; extra == "speaker"
+Requires-Dist: youtokentome; extra == "speaker"
 
 # TalkBank | Batchalign2
 

{batchalign-0.7.3b15 → batchalign-0.7.3b17}/batchalign/formats/chat/utils.py

@@ -43,17 +43,8 @@ def chat_parse_mor(mor_str):
     if mor_str in ENDING_PUNCT:
         return [Morphology(lemma=mor_str, pos="PUNCT", feats="")]
 
-    # JANK handle + forms
-    if "+" in mor_str:
-        pos, rest = mor_str.split("+", maxsplit=1)
-        return [Morphology.model_validate({
-            "lemma": "+"+rest,
-            "pos": pos.strip("|"),
-            "feats": "",
-        })]
-
     try:
-        mors = [i.split("|") for i in re.split("[~$]", mor_str)]
+        mors = [i.split("|", maxsplit=1) for i in re.split("[~$]", mor_str)]
         # TODO epic jank: backwards compatibility check: if a form
         # uses a lot of dashes, its probably because its old-style
         # dash seperated; if it doesn't; it probably is new-style
@@ -67,11 +58,32 @@ def chat_parse_mor(mor_str):
     except:
         raise CHATValidationException(f"mor parser recieved invalid mor string: '{mor_str}'")
 
-    mors = [Morphology.model_validate({
-        "lemma": l,
-        "pos": p,
-        "feats": f,
-    }) for p,l,f in zip(pos, lemmas, feats)]
+
+    mors = []
+    for p,l,f in zip(pos, lemmas, feats):
+        # if "+" not in mor_str:
+        mors.append(Morphology.model_validate({
+            "lemma": l,
+            "pos": p,
+            "feats": f,
+        }))
+        # else:
+        #     breakpoint()
+        #     pos, rest = mor_str.split("+", maxsplit=1)
+
+
+
+    # # JANK handle + forms
+    # if "+" in mor_str:
+    #     pos, rest = mor_str.split("+", maxsplit=1)
+    #     return [Morphology.model_validate({
+    #         "lemma": "+"+rest,
+    #         "pos": pos.strip("|"),
+    #         "feats": "",
+    #     })]
+
+
+    # mors = [ ]
 
     return mors
     

batchalign-0.7.3b17/batchalign/pipelines/morphosyntax/ja/verbforms.py

@@ -0,0 +1,118 @@
+"""
+verbforms.py
+Fix Japanese verb forms.
+"""
+
+def verbform(upos, target, text):
+    if "撮る" in text:
+        return "verb", "撮る"
+    if "貼る" in text:
+        return "verb", "貼る"
+    if "混ぜ" in text:
+        return "verb", "混ぜる"
+    if "釣る" in text:
+        return "verb", "釣る"
+    if "速い" in text and upos == "adj":
+        return "adj", "速い"
+    if "治ま" in text:
+        return "verb", "治まる"
+    if "刺す" in text:
+        return "verb", "刺す"
+    if "降り" in text:
+        return "verb", "降りる"
+    if "降" in text:
+        return "verb", "降る"
+    if "載せ" in text:
+        return "verb", "載せる"
+    if "帰" in text:
+        return "verb", "帰る"
+    if "はい" in text:
+        return "intj", "はい"
+    if "うん" in text:
+        return "intj", "うん"
+    if "おっ" in text:
+        return "intj", "おっ"
+    if "ほら" in text:
+        return "intj", "ほら"
+    if "ヤッホー" in text:
+        return "intj", "ヤッホー"
+    if "ただいま" in text:
+        return "intj", "ただいま"
+    if "あたし" in text:
+        return "pron", "あたし"
+    if "舐め" in text:
+        return "verb", "舐める"
+    if "バツ" in text:
+        return "noun", "バツ"
+    if "ブラシ" in text:
+        return "noun", "ブラシ"
+    if "引き出し" in text:
+        return "noun", "引き出し"
+    if "下さい" in text:
+        return "noun", "下さい"
+    if target in ["シャャミー", "物コャミ"]:
+        return "noun", "クシャミ"
+    if "マヨネーズ" in text:
+        return "noun", "マヨネーズ"
+    if "マヨ" in text:
+        return "noun", "マヨ"
+    if "チップス" in text:
+        return "noun", "チップス"
+    if "ゴロンっ" in text:
+        return "noun", "ゴロンっ"
+    if "モチーンっ" in text:
+        return "noun", "モチーンっ"
+    if "人っ" == text:
+        return "noun", "人"
+    if text == "掻く":
+        return "part", "かい"
+    if "遣" in text and upos == "noun":
+        return "verb", "遣る"
+    if "死" in text:
+        return "verb", "死ぬ"
+    if "立" in text:
+        return "verb", "立つ"
+    if "引" in text:
+        return "verb", "引く"
+    if "出" in text:
+        return "verb", "出す"
+    if "引" in text:
+        return "verb", "引く"
+    if "飲" in text:
+        return "verb", "飲む"
+    if "呼" in text:
+        return "verb", "呼ぶ"
+    if "脱" in text:
+        return "verb", "脱ぐ"
+    if text == "な" and upos == "part":
+        return "aux", "な"
+    if text == "呼ん":
+        return "verb", "呼ぶ"
+    if text == "な" and upos == "aux":
+        return "aux", "な"
+    if text == "だり":
+        return "aux", "たり"
+    if text == "たり":
+        return "aux", "たり"
+    if text == "たら":
+        return "sconj", "たら"
+    if text == "たっ":
+        return "sconj", "たって"
+    # if text == "て" and upos == "sconj":
+    #     return "aux", "て"
+    if text == "なさい" and target == "為さる":
+        return "aux", "為さい"
+    if text == "な" and upos == "part":
+        return "aux", "な"
+    if text == "脱" and upos == "noun":
+        return "verb", "脱"
+    if text == "よう" and upos == "aux":
+        return "aux", "よう"
+    if text == "ろ" and upos == "aux" and target == "為る":
+        return "aux", "ろ"
+    # if upos == "verb" and "る" in target:
+    #     return "verb", target.replace("る","").strip()
+
+    return upos,target
+
+

{batchalign-0.7.3b15 → batchalign-0.7.3b17}/batchalign/pipelines/morphosyntax/ud.py

@@ -237,6 +237,8 @@ def handler__VERB(word, lang=None):
     res = handler(word, lang)
     if "sconj" in res:
         return res
+    elif "verb" not in res:
+        return res
     else:
         return res+flag+stringify_feats(aspect, mood,
                                         tense, polarity, polite,
@@ -266,7 +268,10 @@ def handler__PUNCT(word, lang=None):
         return "noun|da"
     elif re.match(r"^['\w-]+$", word.text): # we match text here because .text is the ultumate content
                                         # instead of the lemma, which maybe entirely weird
-        return f"x|{word.text}"
+        if word.text == "もん":
+            return f"part|{word.text}"
+        else:
+            return f"x|{word.text}"
 
 # Register handlers
 HANDLERS = {
@@ -885,6 +890,7 @@ def morphoanalyze(doc: Document, retokenize:bool, status_hook:callable = None, *
                 retokenized_ut = retokenized_ut.replace(" :", ":")
                 retokenized_ut = retokenized_ut.replace("+ ,", "+,")
                 retokenized_ut = retokenized_ut.replace(": <", ": <")
+                # retokenized_ut = retokenized_ut.replace("[ *", "[*")
                 retokenized_ut = retokenized_ut.replace(" ↑", "↑")
                 retokenized_ut = re.sub(r"@ ?w ?p", "@wp", retokenized_ut)
                 retokenized_ut = retokenized_ut.replace(" @", "@")

batchalign-0.7.3b17/batchalign/version

@@ -0,0 +1,3 @@
+0.7.3-beta.17
+August 21st, 2024
+manual rules with mor

{batchalign-0.7.3b15 → batchalign-0.7.3b17/batchalign.egg-info}/PKG-INFO

@@ -1,16 +1,76 @@
 Metadata-Version: 2.1
 Name: batchalign
-Version: 0.7.3b15
+Version: 0.7.3b17
 Summary: Python Speech Language Sample Analysis
 Author: Brian MacWhinney, Houjun Liu
 Author-email: macw@cmu.edu, houjun@cmu.edu
 Classifier: Development Status :: 3 - Alpha
 Classifier: Topic :: Utilities
 Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pydantic>=2.4
+Requires-Dist: nltk>=3.8
+Requires-Dist: praatio<6.1.0,>=6.0.0
+Requires-Dist: torch<2.2.0,>=2.0.1
+Requires-Dist: torchaudio<2.2.0,>=2.1.0
+Requires-Dist: pyAudioAnalysis==0.3.14
+Requires-Dist: hmmlearn==0.3.0
+Requires-Dist: eyed3~=0.9.7
+Requires-Dist: pydub<0.26.0,>=0.25.1
+Requires-Dist: imblearn
+Requires-Dist: plotly>=5.18.0
+Requires-Dist: transformers~=4.37
+Requires-Dist: tokenizers>=0.14.1
+Requires-Dist: pycountry>=22.3
+Requires-Dist: stanza>=1.7
+Requires-Dist: scipy~=1.11
+Requires-Dist: rev_ai>=2.18.0
+Requires-Dist: rich~=13.6
+Requires-Dist: click~=8.1
+Requires-Dist: matplotlib<4.0.0,>=3.8.0
+Requires-Dist: pyfiglet==1.0.2
+Requires-Dist: soundfile~=0.12.0
+Requires-Dist: rich-click>=1.7.0
+Requires-Dist: typing-extensions
 Provides-Extra: dev
+Requires-Dist: pytest; extra == "dev"
 Provides-Extra: train
+Requires-Dist: accelerate~=0.27; extra == "train"
 Provides-Extra: speaker
-License-File: LICENSE
+Requires-Dist: nemo-toolkit~=1.21.0; extra == "speaker"
+Requires-Dist: omegaconf~=2.3.0; extra == "speaker"
+Requires-Dist: pydub~=0.25.0; extra == "speaker"
+Requires-Dist: braceexpand; extra == "speaker"
+Requires-Dist: editdistance; extra == "speaker"
+Requires-Dist: g2p_en; extra == "speaker"
+Requires-Dist: ipywidgets; extra == "speaker"
+Requires-Dist: jiwer; extra == "speaker"
+Requires-Dist: kaldi-python-io; extra == "speaker"
+Requires-Dist: kaldiio; extra == "speaker"
+Requires-Dist: lhotse>=1.20.0; extra == "speaker"
+Requires-Dist: librosa>=0.10.0; extra == "speaker"
+Requires-Dist: marshmallow; extra == "speaker"
+Requires-Dist: matplotlib; extra == "speaker"
+Requires-Dist: packaging; extra == "speaker"
+Requires-Dist: pyannote.core; extra == "speaker"
+Requires-Dist: pyannote.metrics; extra == "speaker"
+Requires-Dist: pydub; extra == "speaker"
+Requires-Dist: pyloudnorm; extra == "speaker"
+Requires-Dist: resampy; extra == "speaker"
+Requires-Dist: ruamel.yaml; extra == "speaker"
+Requires-Dist: scipy>=0.14; extra == "speaker"
+Requires-Dist: soundfile; extra == "speaker"
+Requires-Dist: sox; extra == "speaker"
+Requires-Dist: texterrors; extra == "speaker"
+Requires-Dist: hydra-core<=1.3.2,>1.3; extra == "speaker"
+Requires-Dist: omegaconf<=2.3; extra == "speaker"
+Requires-Dist: pytorch-lightning>=2.2.1; extra == "speaker"
+Requires-Dist: torchmetrics>=0.11.0; extra == "speaker"
+Requires-Dist: transformers>=4.36.0; extra == "speaker"
+Requires-Dist: wandb; extra == "speaker"
+Requires-Dist: webdataset>=0.2.86; extra == "speaker"
+Requires-Dist: sentencepiece; extra == "speaker"
+Requires-Dist: youtokentome; extra == "speaker"
 
 # TalkBank | Batchalign2
 

batchalign-0.7.3b15/batchalign/pipelines/morphosyntax/ja/verbforms.py

@@ -1,56 +0,0 @@
-"""
-verbforms.py
-Fix Japanese verb forms.
-"""
-
-def verbform(upos, target, text):
-    if "遣" in text and upos == "noun":
-        return "verb", "遣る"
-    if "死" in text:
-        return "verb", "死ぬ"
-    if "立" in text:
-        return "verb", "立つ"
-    if "引" in text:
-        return "verb", "引く"
-    if "出" in text:
-        return "verb", "出す"
-    if "引" in text:
-        return "verb", "引く"
-    if "飲" in text:
-        return "verb", "飲む"
-    if "呼" in text:
-        return "verb", "呼ぶ"
-    if "脱" in text:
-        return "verb", "脱ぐ"
-    if text == "な" and upos == "part":
-        return "aux", "な"
-    if text == "呼ん":
-        return "verb", "呼ぶ"
-    if text == "な" and upos == "aux":
-        return "aux", "な"
-    if text == "だり":
-        return "aux", "たり"
-    if text == "たり":
-        return "aux", "たり"
-    if text == "たら":
-        return "sconj", "たら"
-    if text == "たっ":
-        return "sconj", "たって"
-    # if text == "て" and upos == "sconj":
-    #     return "aux", "て"
-    if text == "なさい" and target == "為さる":
-        return "aux", "為さい"
-    if text == "な" and upos == "part":
-        return "aux", "な"
-    if text == "脱" and upos == "noun":
-        return "verb", "脱"
-    if text == "よう" and upos == "aux":
-        return "aux", "よう"
-    if text == "ろ" and upos == "aux" and target == "為る":
-        return "aux", "ろ"
-    # if upos == "verb" and "る" in target:
-    #     return "verb", target.replace("る","").strip()
-
-    return upos,target
-
-

batchalign-0.7.3b15/batchalign/version

@@ -1,3 +0,0 @@
-0.7.3-beta.15
-July 29th, 2024
-Correct Unicode Escapes?