BatchalignHK 0.7.17.post17__tar.gz → 0.7.17.post19__tar.gz

{batchalignhk-0.7.17.post17 → batchalignhk-0.7.17.post19}/BatchalignHK.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.2
 Name: BatchalignHK
-Version: 0.7.17.post17
+Version: 0.7.17.post19
 Summary: Python Speech Language Sample Analysis
 Author: Brian MacWhinney, Houjun Liu
 Author-email: macw@cmu.edu, houjun@cmu.edu
@@ -48,7 +48,6 @@ Dynamic: author-email
 Dynamic: classifier
 Dynamic: description
 Dynamic: description-content-type
-Dynamic: license-file
 Dynamic: provides-extra
 Dynamic: requires-dist
 Dynamic: summary

{batchalignhk-0.7.17.post17 → batchalignhk-0.7.17.post19}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.2
 Name: BatchalignHK
-Version: 0.7.17.post17
+Version: 0.7.17.post19
 Summary: Python Speech Language Sample Analysis
 Author: Brian MacWhinney, Houjun Liu
 Author-email: macw@cmu.edu, houjun@cmu.edu
@@ -48,7 +48,6 @@ Dynamic: author-email
 Dynamic: classifier
 Dynamic: description
 Dynamic: description-content-type
-Dynamic: license-file
 Dynamic: provides-extra
 Dynamic: requires-dist
 Dynamic: summary

{batchalignhk-0.7.17.post17 → batchalignhk-0.7.17.post19}/batchalign/cli/dispatch.py

@@ -66,12 +66,16 @@ def _dispatch(command, lang, num_speakers,
 
     if kwargs.get("data"):
         url = kwargs.get("data")
-        url = urlparse(url)
-        if url.scheme == "":
-            url = url._replace(scheme="http")
-        base = os.path.basename(url.path)
-        files.append(url)
-        outputs.append(os.path.join(out_dir, base))
+        with open(url.strip()) as data:
+            data = data.readlines()
+        data = [i.strip() for i in data if i.strip() != ""]
+        for url in data:
+            url = urlparse(url)
+            if url.scheme == "":
+                url = url._replace(scheme="http")
+            base = os.path.basename(url.path)
+            files.append(url)
+            outputs.append(os.path.join(out_dir, base))
 
     for basedir, _, fs in os.walk(in_dir):
         for f in fs:

{batchalignhk-0.7.17.post17 → batchalignhk-0.7.17.post19}/batchalign/models/utterance/cantonese_infer.py

@@ -19,6 +19,9 @@ from transformers import DataCollatorForTokenClassification
 # tqdm
 from tqdm import tqdm
 
+import logging
+L = logging.getLogger("batchalign")
+
 # seed device and tokens
 DEVICE = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')
 
@@ -34,8 +37,8 @@ class BertCantoneseUtteranceModel(object):
 
         # eval mode
         self.model.eval()
-        print(f"Model and tokenizer initialized on device: {DEVICE}")
-        print(f"Max length set to {self.max_length} with overlap of {self.overlap}")
+        L.debug(f"Model and tokenizer initialized on device: {DEVICE}")
+        L.debug(f"Max length set to {self.max_length} with overlap of {self.overlap}")
 
     def __call__(self, passage):
         # Step 1: Clean up passage
@@ -79,14 +82,14 @@ class BertCantoneseUtteranceModel(object):
                 break
         
         # Debugging: Print number of chunks and their content
-        print(f"Created {len(chunks)} chunks based on keywords.")
+        L.debug(f"Created {len(chunks)} chunks based on keywords.")
         for i, chunk in enumerate(chunks):
-            print(f"Chunk {i + 1}: {chunk[:100]}...")  # Print the first 100 characters of each chunk
+            L.debug(f"Chunk {i + 1}: {chunk[:100]}...")  # Print the first 100 characters of each chunk
         
         # Step 3: Process each chunk and restore punctuation
         final_passage = []
         for chunk_index, chunk in enumerate(chunks):
-            print(f"Processing chunk {chunk_index + 1}/{len(chunks)}...")
+            L.debug(f"Processing chunk {chunk_index + 1}/{len(chunks)}...")
 
             # Step 3.1: Split chunk by characters (Chinese tokenization)
             tokenized_chunk = list(chunk)  # Simply split by characters for Chinese text
@@ -103,7 +106,7 @@ class BertCantoneseUtteranceModel(object):
                 # Pass it through the model
                 res = self.model(**tokd).logits
             except Exception as e:
-                print(f"Error during model inference: {e}")
+                L.error(f"Error during model inference: {e}")
                 return []
 
             # Argmax for classification
@@ -152,7 +155,7 @@ class BertCantoneseUtteranceModel(object):
         # Step 4: Join processed chunks together into the final passage
         final_passage = ' '.join(final_passage)
 
-        print("Text processing completed. Generating final output...")
+        L.info("Text processing completed. Generating final output...")
         
         # Optionally, tokenize the final text into sentences based on punctuation
         def custom_sent_tokenize(text):
@@ -164,12 +167,12 @@ class BertCantoneseUtteranceModel(object):
             parts = re.split(sentence_endings, text) 
             
             # Debug: Output the parts after splitting
-            print(f"Parts after splitting: {parts}")
+            L.debug(f"Parts after splitting: {parts}")
         
             # Combine parts and punctuation together
             for i in range(0, len(parts) - 1, 2):
                 sentence = parts[i] + parts[i + 1]  # Join sentence with punctuation
-                print(f"Sentence formed: {sentence}")  # Debug: Output the current sentence
+                L.debug(f"Sentence formed: {sentence}")  # Debug: Output the current sentence
                 
                 if sentence.strip():  # Only add non-empty sentences (check for non-whitespace content)
                     split_passage.append(sentence)
@@ -177,18 +180,18 @@ class BertCantoneseUtteranceModel(object):
             # If the last part doesn't have punctuation, we handle it here
             if len(parts) % 2 != 0:  # If there's no punctuation at the end
                 last_part = parts[-1].strip()
-                print(f"Last part without punctuation: {last_part}")  # Debug: Output the last part
+                L.debug(f"Last part without punctuation: {last_part}")  # Debug: Output the last part
                 
                 if last_part:  # Only add non-empty sentences
                     split_passage.append(last_part)
         
             # Final output
-            print(f"Final split passage: {split_passage}")
+            L.debug(f"Final split passage: {split_passage}")
             return split_passage
             
         split_passage = custom_sent_tokenize(final_passage)
 
         # Debugging: Output the sentences after splitting
-        print(f"Final sentences: {split_passage}")
+        L.debug(f"Final sentences: {split_passage}")
 
         return split_passage

{batchalignhk-0.7.17.post17 → batchalignhk-0.7.17.post19}/batchalign/pipelines/morphosyntax/ud.py

@@ -462,7 +462,7 @@ def parse_sentence(sentence, delimiter=".", special_forms=[], lang="$nospecial$"
             # specivl forms: recall the special form marker is xbxxx
             if "xbxxx" in word.text.strip():
                 form = special_forms.pop(0)
-                mor.append(f"x|{form.strip().replace(',', 'cm')}")
+                mor.append(f"{form[1].strip()}|{form[0].strip().replace(',', 'cm')}")
                 special_form_ids.append(word.id)
             else:
                 mor.append(mor_word)
@@ -555,7 +555,6 @@ def parse_sentence(sentence, delimiter=".", special_forms=[], lang="$nospecial$"
                           # add a deliminator
         mor_str = mor_str + " " + delimiter
 
-
     mor_str = mor_str.replace("<UNK>", "")
     gra_str = gra_str.replace("<UNK>", "")
 
@@ -843,7 +842,7 @@ def morphoanalyze(doc: Document, retokenize:bool, status_hook:callable = None, *
         special_forms_cleaned = []
         for form in special_forms:
             line_cut = line_cut.replace(form, "xbxxx")
-            special_forms_cleaned.append(re.sub(r"@[\w\:]+", "", form).strip())
+            special_forms_cleaned.append(form.split("@"))
 
         # if line cut is still nothing, we get very angry
         if line_cut == "":
@@ -942,7 +941,7 @@ def morphoanalyze(doc: Document, retokenize:bool, status_hook:callable = None, *
                         if ut[i.payload].text != ",":
                             ut[i.payload].morphology = [Morphology(
                                 lemma = sents[0].tokens[i.payload].text if len(sents) > 0 and len(sents[0].tokens) > i.payload and sents[0].tokens[i.payload].text != "xbxxx" else ut[i.payload].text,
-                                pos = "x",
+                                pos = ut[i.payload].morphology[0].pos if (ut[i.payload].morphology and len(ut[i.payload].morphology) > 0) else "x",
                                 feats = ""
                             )]
                 poses = [i.morphology[0].pos.upper() for i in ut
@@ -999,6 +998,7 @@ def morphoanalyze(doc: Document, retokenize:bool, status_hook:callable = None, *
                     content.dependency = form.dependency
 
         except Exception as e:
+            raise e
             pass
             # warnings.warn(f"Utterance failed parsing, skipping ud tagging... line='{line}', error='{e}'.\n")
 

batchalignhk-0.7.17.post19/batchalign/version

@@ -0,0 +1,3 @@
+0.7.17-post.19
+April 06th, 2025
+Japanese form fixes

batchalignhk-0.7.17.post17/batchalign/version

@@ -1,3 +0,0 @@
-0.7.17-post.17
-March 26th, 2025
-better tencent ASR