BatchalignHK 0.7.19.post7__tar.gz → 0.7.19.post9__tar.gz

{batchalignhk-0.7.19.post7 → batchalignhk-0.7.19.post9}/BatchalignHK.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.2
 Name: BatchalignHK
-Version: 0.7.19.post7
+Version: 0.7.19.post9
 Summary: Python Speech Language Sample Analysis
 Author: Brian MacWhinney, Houjun Liu
 Author-email: macw@cmu.edu, houjun@cmu.edu
@@ -50,7 +50,6 @@ Dynamic: author-email
 Dynamic: classifier
 Dynamic: description
 Dynamic: description-content-type
-Dynamic: license-file
 Dynamic: provides-extra
 Dynamic: requires-dist
 Dynamic: summary

{batchalignhk-0.7.19.post7 → batchalignhk-0.7.19.post9}/BatchalignHK.egg-info/SOURCES.txt

@@ -115,6 +115,7 @@ batchalign/tests/pipelines/cleanup/test_disfluency.py
 batchalign/tests/pipelines/cleanup/test_parse_support.py
 batchalign/tests/pipelines/fa/test_fa_pipeline.py
 batchalign/utils/__init__.py
+batchalign/utils/abbrev.py
 batchalign/utils/config.py
 batchalign/utils/dp.py
 batchalign/utils/names.py

{batchalignhk-0.7.19.post7 → batchalignhk-0.7.19.post9}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.2
 Name: BatchalignHK
-Version: 0.7.19.post7
+Version: 0.7.19.post9
 Summary: Python Speech Language Sample Analysis
 Author: Brian MacWhinney, Houjun Liu
 Author-email: macw@cmu.edu, houjun@cmu.edu
@@ -50,7 +50,6 @@ Dynamic: author-email
 Dynamic: classifier
 Dynamic: description
 Dynamic: description-content-type
-Dynamic: license-file
 Dynamic: provides-extra
 Dynamic: requires-dist
 Dynamic: summary

{batchalignhk-0.7.19.post7 → batchalignhk-0.7.19.post9}/batchalign/document.py

@@ -11,6 +11,7 @@ from pathlib import Path
 
 from batchalign.errors import *
 from batchalign.constants import *
+from batchalign.utils.abbrev import abbrev
 
 import re
 
@@ -223,6 +224,11 @@ class Utterance(BaseModel):
         t = re.sub(r",", " , ", t.strip()).strip()
         t = re.sub(r" +", " ", t.strip()).strip()
         t = t.replace("+ ,", "+,").strip()
+
+        abbrevs =  [" " .join(list(i)) for i in abbrev]
+        for i in abbrevs:
+            t = t.replace(i, i.replace(" ", ""))
+
         return t
 
     def __repr__(self):

{batchalignhk-0.7.19.post7 → batchalignhk-0.7.19.post9}/batchalign/pipelines/asr/tencent.py

@@ -21,8 +21,8 @@ import tempfile
 import pycountry
 import numpy as np
 import soundfile as sf
-from pydub import AudioSegment
-from pydub.effects import normalize
+# from pydub import AudioSegment
+# from pydub.effects import normalize
 import base64
 from tencentcloud.common.credential import Credential
 from tencentcloud.asr.v20190614.asr_client import AsrClient, models
@@ -30,9 +30,9 @@ from tencentcloud.asr.v20190614.asr_client import AsrClient, models
 import asyncio
 import tempfile
 import os
-from pydub import AudioSegment
-from pydub.effects import normalize
-from pydub.exceptions import CouldntDecodeError
+# from pydub import AudioSegment
+# from pydub.effects import normalize
+# from pydub.exceptions import CouldntDecodeError
 
 
 import logging
@@ -77,66 +77,6 @@ class TencentEngine(BatchalignEngine):
             L.debug("Done.")
         else:
             self.__engine = None
-
-    def __preprocess_audio(self, input_path):
-        """Enhanced audio preprocessing for low-volume speech"""
-        try:
-            L.info(f"Optimizing audio for ASR: {input_path}")
-            
-            # read the audio file
-            audio = AudioSegment.from_file(input_path)
-
-            audio = audio.set_channels(1)  
-            audio = audio.set_frame_rate(16000)  
-
-            
-            audio = audio.compress_dynamic_range(
-                threshold=-40, 
-                ratio=3,
-                attack=5, 
-                release=100
-            )
-            audio = audio.low_pass_filter(4000)  # filter out high frequencies
-            audio = audio.normalize(headroom=2)  # keep the headroom
-            audio = audio.compress_dynamic_range(
-                threshold=-55,
-                ratio=6,
-                attack=15,
-                release=200
-            )
-
-            # enhance low volume
-            audio = audio.high_pass_filter(80)
-            boosted = audio.high_pass_filter(1000).apply_gain(+4)
-            audio = audio.overlay(boosted)
-
-            if L.level <= logging.DEBUG:
-                self.__print_audio_stats(audio)
-
-            # output to a temporary file
-            temp_fd, temp_path = tempfile.mkstemp(suffix=".mp3")
-            os.close(temp_fd)
-            audio.export(
-                temp_path,
-                format="mp3",
-                codec="libmp3lame",
-                bitrate="96k",
-                tags={"title": "BA_Optimized"},
-                parameters=[
-                    "-compression_level", "2",
-                    "-reservoir", "0",
-                    "-joint_stereo", "0"
-                ]
-            )
-
-            return temp_path
-
-        except CouldntDecodeError:
-            L.error(f"Audio decoding failed: {input_path}")
-            return input_path
-        except Exception as e:
-            L.error(f"Audio processing error: {str(e)}")
-            return input_path
         
     def replace_cantonese_words(self, word):
         """Function to replace Cantonese words with custom replacements."""
@@ -176,13 +116,15 @@ class TencentEngine(BatchalignEngine):
         lang = self.__lang
         client = self.__client
 
-        processed_path = self.__preprocess_audio(f)
-        audio = AudioSegment.from_file(processed_path)
+        # processed_path = self.__preprocess_audio(f)
+        # audio = AudioSegment.from_file(processed_path)
         
         try:
             L.info(f"Uploading '{pathlib.Path(f).stem}'...")
-            with open(processed_path, "rb") as audio_file:
-                encoded_string = base64.b64encode(audio_file.read())
+            # we will send the file for processing
+            if not str(f).startswith("http"):
+                with open(f, "rb") as image_file:
+                    encoded_string = base64.b64encode(image_file.read())
 
             req = models.CreateRecTaskRequest()
             if lang in {'zho', 'yue', 'wuu', 'nan','hak'}:
@@ -192,9 +134,12 @@ class TencentEngine(BatchalignEngine):
             req.ResTextFormat = 1
             req.SpeakerDiarization = 1
             req.ChannelNum = 1
-            req.Data = encoded_string.decode('ascii')
-            req.SourceType = 1
-            
+            if not str(f).startswith("http"):
+                req.Data = encoded_string.decode('ascii')
+                req.SourceType = 1
+            else:
+                req.Url = f
+                req.SourceType = 0
             resp = client.CreateRecTask(req)
 
             L.info(f"Tencent is transcribing '{pathlib.Path(f).stem}'...")

{batchalignhk-0.7.19.post7 → batchalignhk-0.7.19.post9}/batchalign/pipelines/morphosyntax/ud.py

@@ -711,6 +711,7 @@ def adlist_postprocessor(i, lang, adlist):
 
 ######
 def morphoanalyze(doc: Document, retokenize:bool, skipmultilang:bool, status_hook:callable = None, **kwargs):
+
     L.debug("Starting Stanza...")
     inputs = []
 

batchalignhk-0.7.19.post9/batchalign/utils/abbrev.py

@@ -0,0 +1,182 @@
+abbrev = [
+    "FBI",
+    "CIA",
+    "NSA",
+    "NATO",
+    "UN",
+    "WHO",
+    "NASA",
+    "CDC",
+    "IRS",
+    "EPA",
+    "HTTP",
+    "URL",
+    "HTML",
+    "CSS",
+    "API",
+    "IP",
+    "DNS",
+    "SQL",
+    "USB",
+    "VPN",
+    "ATT",
+    "AT&T",
+    "CEO",
+    "CFO",
+    "COO",
+    "IPO",
+    "ROI",
+    "GDP",
+    "LLC",
+    "HR",
+    "M&",
+    "KPI",
+    "GPA",
+    "SAT",
+    "ACT",
+    "MBA",
+    "PhD",
+    "BA",
+    "MA",
+    "STEM",
+    "ESL",
+    "GED",
+    "AWOL",
+    "MIA",
+    "POW",
+    "IED",
+    "UAV",
+    "RPG",
+    "NATO",
+    "SEAL",
+    "JAG",
+    "ROTC",
+    "CERN",
+    "GMO",
+    "H2O",
+    "CO2",
+    "UV",
+    "IR",
+    "AI",
+    "VR",
+    "AR",
+    "NPR",
+    "BBC",
+    "MTV",
+    "CNN",
+    "HBO",
+    "ESPN",
+    "TMZ",
+    "AMC",
+    "IMAX",
+    "WWE",
+    "ASAP",
+    "DIY",
+    "ETA",
+    "RSVP",
+    "FYI",
+    "LOL",
+    "BRB",
+    "IDK",
+    "BTW",
+    "TMI",
+    "PBJ",
+    "AIDS",
+    "HIV",
+    "ADHD",
+    "COPD",
+    "PTSD",
+    "CHF",
+    "CAD",
+    "TB",
+    "UTI",
+    "GERD",
+    "MRI",
+    "CT",
+    "ECG",
+    "EEG",
+    "CBC",
+    "BMP",
+    "ABG",
+    "PFT",
+    "FOBT",
+    "ENT",
+    "OB",
+    "PCP",
+    "ICU",
+    "NICU",
+    "ER",
+    "OR",
+    "PT",
+    "OT",
+    "EM",
+    "OTC",
+    "NSAID",
+    "IV",
+    "IM",
+    "SC",
+    "PRN",
+    "BID",
+    "TID",
+    "QID",
+    "NPO",
+    "CNS",
+    "PNS",
+    "GI",
+    "GU",
+    "CV",
+    "MSK",
+    "ENT",
+    "BMI",
+    "BMR",
+    "BP",
+    "WBC",
+    "RBC",
+    "HGB",
+    "HCT",
+    "PLT",
+    "ESR",
+    "CRP",
+    "LFT",
+    "TFT",
+    "INR",
+    "MMR",
+    "DPT",
+    "HPV",
+    "Tdap",
+    "BCG",
+    "IPV",
+    "HBV",
+    "HAV",
+    "HCV",
+    "RSV",
+    "SOAP",
+    "DNR",
+    "AMA",
+    "LOS",
+    "EHR",
+    "EMR",
+    "ICD",
+    "CPT",
+    "HIPAA",
+    "HR",
+    "RR",
+    "SpO2",
+    "MAP",
+    "GFR",
+    "A1C",
+    "LDL",
+    "HDL",
+    "TG",
+    "BUN",
+    "SIDS",
+    "DVT",
+    "PE",
+    "ARDS",
+    "SLE",
+    "RA",
+    "TIA",
+    "CVA",
+    "ALS",
+    "MS",
+]

batchalignhk-0.7.19.post9/batchalign/version

@@ -0,0 +1,3 @@
+0.7.19-post.9
+May 24th, 2025
+reverts file only prep changes

batchalignhk-0.7.19.post7/batchalign/version

@@ -1,3 +0,0 @@
-0.7.19-post.7
-May 20th, 2025
-fixes for ASR