BatchalignHK 0.7.17.post16__tar.gz → 0.7.17.post18__tar.gz

{batchalignhk-0.7.17.post16 → batchalignhk-0.7.17.post18}/BatchalignHK.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: BatchalignHK
-Version: 0.7.17.post16
+Version: 0.7.17.post18
 Summary: Python Speech Language Sample Analysis
 Author: Brian MacWhinney, Houjun Liu
 Author-email: macw@cmu.edu, houjun@cmu.edu
@@ -15,6 +15,7 @@ Requires-Dist: torch>=2.6.0
 Requires-Dist: torchaudio
 Requires-Dist: hmmlearn==0.3.0
 Requires-Dist: eyed3
+Requires-Dist: opencc-python-reimplemented
 Requires-Dist: pydub
 Requires-Dist: imblearn
 Requires-Dist: plotly>=5.3.0
@@ -47,6 +48,7 @@ Dynamic: author-email
 Dynamic: classifier
 Dynamic: description
 Dynamic: description-content-type
+Dynamic: license-file
 Dynamic: provides-extra
 Dynamic: requires-dist
 Dynamic: summary

{batchalignhk-0.7.17.post16 → batchalignhk-0.7.17.post18}/BatchalignHK.egg-info/requires.txt

@@ -5,6 +5,7 @@ torch>=2.6.0
 torchaudio
 hmmlearn==0.3.0
 eyed3
+opencc-python-reimplemented
 pydub
 imblearn
 plotly>=5.3.0

{batchalignhk-0.7.17.post16 → batchalignhk-0.7.17.post18}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: BatchalignHK
-Version: 0.7.17.post16
+Version: 0.7.17.post18
 Summary: Python Speech Language Sample Analysis
 Author: Brian MacWhinney, Houjun Liu
 Author-email: macw@cmu.edu, houjun@cmu.edu
@@ -15,6 +15,7 @@ Requires-Dist: torch>=2.6.0
 Requires-Dist: torchaudio
 Requires-Dist: hmmlearn==0.3.0
 Requires-Dist: eyed3
+Requires-Dist: opencc-python-reimplemented
 Requires-Dist: pydub
 Requires-Dist: imblearn
 Requires-Dist: plotly>=5.3.0
@@ -47,6 +48,7 @@ Dynamic: author-email
 Dynamic: classifier
 Dynamic: description
 Dynamic: description-content-type
+Dynamic: license-file
 Dynamic: provides-extra
 Dynamic: requires-dist
 Dynamic: summary

{batchalignhk-0.7.17.post16 → batchalignhk-0.7.17.post18}/batchalign/cli/dispatch.py

@@ -66,12 +66,16 @@ def _dispatch(command, lang, num_speakers,
 
     if kwargs.get("data"):
         url = kwargs.get("data")
-        url = urlparse(url)
-        if url.scheme == "":
-            url = url._replace(scheme="http")
-        base = os.path.basename(url.path)
-        files.append(url)
-        outputs.append(os.path.join(out_dir, base))
+        with open(url.strip()) as data:
+            data = data.readlines()
+        data = [i.strip() for i in data if i.strip() != ""]
+        for url in data:
+            url = urlparse(url)
+            if url.scheme == "":
+                url = url._replace(scheme="http")
+            base = os.path.basename(url.path)
+            files.append(url)
+            outputs.append(os.path.join(out_dir, base))
 
     for basedir, _, fs in os.walk(in_dir):
         for f in fs:

{batchalignhk-0.7.17.post16 → batchalignhk-0.7.17.post18}/batchalign/models/utterance/cantonese_infer.py

@@ -19,6 +19,9 @@ from transformers import DataCollatorForTokenClassification
 # tqdm
 from tqdm import tqdm
 
+import logging
+L = logging.getLogger("batchalign")
+
 # seed device and tokens
 DEVICE = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')
 
@@ -34,8 +37,8 @@ class BertCantoneseUtteranceModel(object):
 
         # eval mode
         self.model.eval()
-        print(f"Model and tokenizer initialized on device: {DEVICE}")
-        print(f"Max length set to {self.max_length} with overlap of {self.overlap}")
+        L.debug(f"Model and tokenizer initialized on device: {DEVICE}")
+        L.debug(f"Max length set to {self.max_length} with overlap of {self.overlap}")
 
     def __call__(self, passage):
         # Step 1: Clean up passage
@@ -79,14 +82,14 @@ class BertCantoneseUtteranceModel(object):
                 break
         
         # Debugging: Print number of chunks and their content
-        print(f"Created {len(chunks)} chunks based on keywords.")
+        L.debug(f"Created {len(chunks)} chunks based on keywords.")
         for i, chunk in enumerate(chunks):
-            print(f"Chunk {i + 1}: {chunk[:100]}...")  # Print the first 100 characters of each chunk
+            L.debug(f"Chunk {i + 1}: {chunk[:100]}...")  # Print the first 100 characters of each chunk
         
         # Step 3: Process each chunk and restore punctuation
         final_passage = []
         for chunk_index, chunk in enumerate(chunks):
-            print(f"Processing chunk {chunk_index + 1}/{len(chunks)}...")
+            L.debug(f"Processing chunk {chunk_index + 1}/{len(chunks)}...")
 
             # Step 3.1: Split chunk by characters (Chinese tokenization)
             tokenized_chunk = list(chunk)  # Simply split by characters for Chinese text
@@ -103,7 +106,7 @@ class BertCantoneseUtteranceModel(object):
                 # Pass it through the model
                 res = self.model(**tokd).logits
             except Exception as e:
-                print(f"Error during model inference: {e}")
+                L.error(f"Error during model inference: {e}")
                 return []
 
             # Argmax for classification
@@ -152,7 +155,7 @@ class BertCantoneseUtteranceModel(object):
         # Step 4: Join processed chunks together into the final passage
         final_passage = ' '.join(final_passage)
 
-        print("Text processing completed. Generating final output...")
+        L.info("Text processing completed. Generating final output...")
         
         # Optionally, tokenize the final text into sentences based on punctuation
         def custom_sent_tokenize(text):
@@ -164,12 +167,12 @@ class BertCantoneseUtteranceModel(object):
             parts = re.split(sentence_endings, text) 
             
             # Debug: Output the parts after splitting
-            print(f"Parts after splitting: {parts}")
+            L.debug(f"Parts after splitting: {parts}")
         
             # Combine parts and punctuation together
             for i in range(0, len(parts) - 1, 2):
                 sentence = parts[i] + parts[i + 1]  # Join sentence with punctuation
-                print(f"Sentence formed: {sentence}")  # Debug: Output the current sentence
+                L.debug(f"Sentence formed: {sentence}")  # Debug: Output the current sentence
                 
                 if sentence.strip():  # Only add non-empty sentences (check for non-whitespace content)
                     split_passage.append(sentence)
@@ -177,18 +180,18 @@ class BertCantoneseUtteranceModel(object):
             # If the last part doesn't have punctuation, we handle it here
             if len(parts) % 2 != 0:  # If there's no punctuation at the end
                 last_part = parts[-1].strip()
-                print(f"Last part without punctuation: {last_part}")  # Debug: Output the last part
+                L.debug(f"Last part without punctuation: {last_part}")  # Debug: Output the last part
                 
                 if last_part:  # Only add non-empty sentences
                     split_passage.append(last_part)
         
             # Final output
-            print(f"Final split passage: {split_passage}")
+            L.debug(f"Final split passage: {split_passage}")
             return split_passage
             
         split_passage = custom_sent_tokenize(final_passage)
 
         # Debugging: Output the sentences after splitting
-        print(f"Final sentences: {split_passage}")
+        L.debug(f"Final sentences: {split_passage}")
 
         return split_passage

{batchalignhk-0.7.17.post16 → batchalignhk-0.7.17.post18}/batchalign/pipelines/asr/tencent.py

@@ -12,6 +12,9 @@ from batchalign.errors import *
 
 from batchalign.models import BertUtteranceModel, BertCantoneseUtteranceModel, resolve
 
+from opencc import OpenCC
+cc = OpenCC('s2hk')
+
 import time
 import pathlib
 import pycountry
@@ -113,7 +116,7 @@ class TencentEngine(BatchalignEngine):
                     "type": "text",
                     "ts": (j.OffsetStartMs+start)/1000,
                     "end_ts": (j.OffsetEndMs+start)/1000,
-                    "value": j.Word
+                    "value": cc.convert(j.Word)
                 })
             turns.append({
                 "elements": turn,

batchalignhk-0.7.17.post18/batchalign/version

@@ -0,0 +1,3 @@
+0.7.17-post.18
+March 26th, 2025
+cantonese ASR with general fixes for inputting a list

{batchalignhk-0.7.17.post16 → batchalignhk-0.7.17.post18}/setup.py

@@ -34,6 +34,7 @@ setup(
         # "pyAudioAnalysis",
         "hmmlearn==0.3.0",
         "eyed3",
+        "opencc-python-reimplemented",
         "pydub",
         "imblearn",
         "plotly>=5.3.0",

batchalignhk-0.7.17.post16/batchalign/version

@@ -1,3 +0,0 @@
-0.7.17-post.16
-March 26th, 2025
-better tencent ASR