batchalign 0.8.2.post7__tar.gz → 0.8.2.post9__tar.gz

{batchalign-0.8.2.post7/batchalign.egg-info → batchalign-0.8.2.post9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: batchalign
-Version: 0.8.2.post7
+Version: 0.8.2.post9
 Summary: Python Speech Language Sample Analysis
 Author: Brian MacWhinney, Houjun Liu
 Author-email: macw@cmu.edu, houjun@cmu.edu
@@ -42,6 +42,7 @@ Requires-Dist: rich-click>=1.7.0
 Requires-Dist: typing-extensions
 Requires-Dist: num2words
 Requires-Dist: tiktoken
+Requires-Dist: pycantonese
 Requires-Dist: blobfile
 Requires-Dist: sentencepiece
 Requires-Dist: googletrans

{batchalign-0.8.2.post7 → batchalign-0.8.2.post9}/batchalign/cli/cli.py

@@ -348,6 +348,32 @@ def utseg(ctx, in_dir, out_dir, lang, num_speakers, **kwargs):
               in_dir, out_dir,
               loader, writer, C)
 
+#################### SEGMENT ################################
+
+@batchalign.command()
+@common_options
+@click.option("--lang",
+              help="sample language in three-letter ISO 3166-1 alpha-3 code",
+              show_default=True,
+              default="yue",
+              type=str)
+@click.option("--merge-abbrev/--no-merge-abbrev",
+              default=False, help="Merge abbreviations in output. Default: no.")
+@click.pass_context
+def segment(ctx, in_dir, out_dir, lang, **kwargs):
+    """Perform word segmentation on transcripts (Cantonese)."""
+    from batchalign.formats.chat import CHATFile
+
+    def loader(file):
+        return CHATFile(path=os.path.abspath(file)).doc
+
+    def writer(doc, output):
+        CHATFile(doc=doc).write(output, merge_abbrev=kwargs.get("merge_abbrev", False))
+
+    _dispatch("segment", lang, 1, ["cha"], ctx,
+              in_dir, out_dir,
+              loader, writer, C)
+
 #################### BENCHMARK ################################
 
 @batchalign.command()

{batchalign-0.8.2.post7 → batchalign-0.8.2.post9}/batchalign/cli/dispatch.py

@@ -70,6 +70,7 @@ POOL_SAFE_ENGINES = {
     "replacement",
     "ngram",
     "compare_analysis_engine",
+    "cantonese_seg",
 }
 
 warnings.filterwarnings('ignore', category=UserWarning, message='TypedStorage is deprecated')
@@ -234,6 +235,11 @@ def _run_pipeline_for_file(command, pipeline, file, output, loader_info, writer_
         with open(json_path, 'w') as f:
             _json.dump(metrics, f)
 
+    elif command == "segment":
+        doc = CHATFile(path=os.path.abspath(file)).doc
+        doc = pipeline(doc, callback=progress_callback)
+        CHATFile(doc=doc).write(output)
+
     elif command == "opensmile":
         from batchalign.document import Document
         doc = Document.new(media_path=file, lang=local_kwargs.get("lang", kwargs.get("lang", "eng")))
@@ -386,6 +392,7 @@ Cmd2Task = {
     "translate": "translate",
     "opensmile": "opensmile",
     "compare": "compare,compare_analysis",
+    "segment": "segment",
 }
 
 # this is the main runner used by all functions
@@ -406,6 +413,7 @@ def _dispatch(command, lang, num_speakers,
         "benchmark",
         "opensmile",
         "compare",
+        "segment",
     }
     if command in worker_handled:
         # Avoid pickling CLI-local loader/writer functions when the worker

{batchalign-0.8.2.post7 → batchalign-0.8.2.post9}/batchalign/document.py

@@ -27,14 +27,15 @@ class Task(IntEnum):
     DISFLUENCY_ANALYSIS = 6
     RETRACE_ANALYSIS = 7
     UTTERANCE_TIMING_RECOVERY = 8 # "bulletize"
-    FORCED_ALIGNMENT = 9
-    FEATURE_EXTRACT = 10
-    MORPHOSYNTAX = 11
-    COREF = 12
-    WER = 13
-    TRANSLATE = 14
-    COMPARE = 15
-    COMPARE_ANALYSIS = 16
+    WORD_SEGMENTATION = 9
+    FORCED_ALIGNMENT = 10
+    FEATURE_EXTRACT = 11
+    MORPHOSYNTAX = 12
+    COREF = 13
+    WER = 14
+    TRANSLATE = 15
+    COMPARE = 16
+    COMPARE_ANALYSIS = 17
 
 
     DEBUG__G = 0
@@ -51,6 +52,7 @@ TypeMap = {
     Task.SPEAKER_RECOGNITION: TaskType.PROCESSING,
     Task.UTTERANCE_SEGMENTATION: TaskType.PROCESSING,
     Task.UTTERANCE_TIMING_RECOVERY: TaskType.PROCESSING,
+    Task.WORD_SEGMENTATION: TaskType.PROCESSING,
     Task.FORCED_ALIGNMENT: TaskType.PROCESSING,
     Task.MORPHOSYNTAX: TaskType.PROCESSING,
     Task.FEATURE_EXTRACT: TaskType.ANALYSIS,
@@ -73,6 +75,7 @@ TaskFriendlyName = {
     Task.SPEAKER_RECOGNITION: "Speaker Recognition",
     Task.UTTERANCE_SEGMENTATION: "Utterance Segmentation",
     Task.UTTERANCE_TIMING_RECOVERY: "Utterance Timing Recovery",
+    Task.WORD_SEGMENTATION: "Word Segmentation",
     Task.FORCED_ALIGNMENT: "Forced Alignment",
     Task.MORPHOSYNTAX: "Morpho-Syntax",
     Task.FEATURE_EXTRACT: "Feature Extraction",

{batchalign-0.8.2.post7 → batchalign-0.8.2.post9}/batchalign/pipelines/analysis/compare.py

@@ -287,6 +287,8 @@ class CompareEngine(BatchalignEngine):
             for form_idx, form in enumerate(utt.content):
                 if form.text.strip() in MOR_PUNCT + ENDING_PUNCT:
                     continue
+                if _get_pos(form) == "PUNCT":
+                    continue
                 if form.text.strip().lower() in fillers:
                     continue
                 main_info.append((utt_idx, form_idx, form))
@@ -303,7 +305,7 @@ class CompareEngine(BatchalignEngine):
         for utt_idx, utt in enumerate(gold_utterances):
             gold_punct[utt_idx] = []
             for form_idx, form in enumerate(utt.content):
-                if form.text.strip() in MOR_PUNCT + ENDING_PUNCT:
+                if form.text.strip() in MOR_PUNCT + ENDING_PUNCT or _get_pos(form) == "PUNCT":
                     gold_punct[utt_idx].append((form_idx, form))
                     continue
                 if form.text.strip().lower() in fillers:

{batchalign-0.8.2.post7 → batchalign-0.8.2.post9}/batchalign/pipelines/dispatch.py

@@ -16,6 +16,7 @@ DEFAULT_PACKAGES = {
     "asr": "whisper_oai",
     "utr": "whisper_utr",
     "fa": "whisper_fa",
+    "segment": "cantonese_seg",
     "speaker": "pyannote",
     "morphosyntax": "stanza",
     "disfluency": "replacement",
@@ -56,6 +57,8 @@ def resolve_engine_specs(pkg_str, lang, num_speakers=None, **arg_overrides):
     if "fa" in packages:
         if "utr" not in packages:
             packages.append("utr")
+        if "segment" not in packages and lang == "yue":
+            packages.append("segment")
 
     overrides = LANGUAGE_OVERRIDE_PACKAGES.get(lang, {})
     specs = []
@@ -123,6 +126,9 @@ def dispatch_pipeline(pkg_str, lang, num_speakers=None, **arg_overrides):
         elif engine == "ngram":
             from batchalign.pipelines.cleanup import NgramRetraceEngine
             engines.append(NgramRetraceEngine())
+        elif engine == "cantonese_seg":
+            from batchalign.pipelines.segmentation import CantoneseSegmentationEngine
+            engines.append(CantoneseSegmentationEngine())
         elif engine == "whisper_fa":
             from batchalign.pipelines.fa import WhisperFAEngine
             engines.append(WhisperFAEngine())

batchalign-0.8.2.post9/batchalign/pipelines/segmentation/__init__.py

@@ -0,0 +1,15 @@
+"""
+Segmentation Pipeline Module
+
+Provides engines for word segmentation of unsegmented text.
+Uses lazy loading to avoid importing heavy dependencies until needed.
+"""
+
+def __getattr__(name):
+    """Lazy loading for segmentation engines."""
+    if name == 'CantoneseSegmentationEngine':
+        from .cantonese_seg import CantoneseSegmentationEngine
+        return CantoneseSegmentationEngine
+    raise AttributeError(f"module '{__name__}' has no attribute '{name}'")
+
+__all__ = ['CantoneseSegmentationEngine']

batchalign-0.8.2.post9/batchalign/pipelines/segmentation/cantonese_seg.py

@@ -0,0 +1,109 @@
+from batchalign.document import *
+from batchalign.pipelines.base import *
+from batchalign.utils.dp import *
+from batchalign.constants import *
+
+import logging
+L = logging.getLogger("batchalign")
+
+
+class CantoneseSegmentationEngine(BatchalignEngine):
+    tasks = [Task.WORD_SEGMENTATION]
+
+    def __init__(self):
+        import pycantonese
+        self.__segment = pycantonese.segment
+
+    def process(self, doc: Document, **kwargs):
+        # no-op for non-Cantonese
+        if "yue" not in doc.langs:
+            L.debug("CantoneseSegmentation: skipping non-Cantonese document.")
+            return doc
+
+        for ut in doc.content:
+            if not isinstance(ut, Utterance):
+                continue
+
+            # separate regular words from punctuation/features at the end
+            regular_forms = []
+            trailing = []
+            for f in ut.content:
+                if f.type in (TokenType.PUNCT, TokenType.FEAT, TokenType.ANNOT):
+                    trailing.append(f)
+                else:
+                    # if we already started collecting trailing punct,
+                    # but hit another regular token, flush trailing back
+                    if trailing:
+                        regular_forms.extend(trailing)
+                        trailing = []
+                    regular_forms.append(f)
+
+            if not regular_forms:
+                continue
+
+            # combine all regular token text (no spaces for CJK)
+            original_text = "".join(f.text for f in regular_forms)
+            if not original_text.strip():
+                continue
+
+            # run pycantonese segmentation
+            segmented_words = self.__segment(original_text)
+
+            # use character-level DP alignment to map segmented words
+            # back onto original tokens
+            #
+            # reference: chars from original tokens, payload = index into regular_forms
+            ref_targets = []
+            for i, f in enumerate(regular_forms):
+                for ch in f.text:
+                    ref_targets.append(ReferenceTarget(ch, payload=i))
+
+            # payload: chars from segmented words, payload = index into segmented_words
+            pay_targets = []
+            for i, word in enumerate(segmented_words):
+                for ch in word:
+                    pay_targets.append(PayloadTarget(ch, payload=i))
+
+            alignments = align(pay_targets, ref_targets, tqdm=False)
+
+            # build a mapping: segmented word index -> list of original token indices
+            seg_to_orig: dict[int, list[int]] = {}
+            for elem in alignments:
+                if isinstance(elem, Match):
+                    seg_to_orig.setdefault(elem.payload, []).append(elem.reference_payload)
+
+            # construct new Form objects from segmented words
+            new_forms = []
+            for seg_idx, word in enumerate(segmented_words):
+                orig_indices = seg_to_orig.get(seg_idx, [])
+
+                # inherit timing: start from first orig, end from last orig
+                time = None
+                if orig_indices:
+                    first_time = regular_forms[orig_indices[0]].time
+                    last_time = regular_forms[orig_indices[-1]].time
+                    if first_time is not None and last_time is not None:
+                        time = (first_time[0], last_time[1])
+                    elif first_time is not None:
+                        time = first_time
+                    elif last_time is not None:
+                        time = last_time
+
+                # determine token type from the first matched original
+                token_type = TokenType.REGULAR
+                if orig_indices:
+                    token_type = regular_forms[orig_indices[0]].type
+
+                new_forms.append(Form(
+                    text=word,
+                    time=time,
+                    type=token_type,
+                ))
+
+            # re-append trailing punctuation/features
+            new_forms.extend(trailing)
+            ut.content = new_forms
+            # clear cached text so _detokenize() regenerates from new Forms
+            ut.text = None
+
+        return doc

batchalign-0.8.2.post9/batchalign/version

@@ -0,0 +1,3 @@
+0.8.2-post.9
+April 02 2026
+segment

{batchalign-0.8.2.post7 → batchalign-0.8.2.post9/batchalign.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: batchalign
-Version: 0.8.2.post7
+Version: 0.8.2.post9
 Summary: Python Speech Language Sample Analysis
 Author: Brian MacWhinney, Houjun Liu
 Author-email: macw@cmu.edu, houjun@cmu.edu
@@ -42,6 +42,7 @@ Requires-Dist: rich-click>=1.7.0
 Requires-Dist: typing-extensions
 Requires-Dist: num2words
 Requires-Dist: tiktoken
+Requires-Dist: pycantonese
 Requires-Dist: blobfile
 Requires-Dist: sentencepiece
 Requires-Dist: googletrans

{batchalign-0.8.2.post7 → batchalign-0.8.2.post9}/batchalign.egg-info/SOURCES.txt

@@ -107,6 +107,8 @@ batchalign/pipelines/morphosyntax/fr/case.py
 batchalign/pipelines/morphosyntax/ja/verbforms.py
 batchalign/pipelines/opensmile/__init__.py
 batchalign/pipelines/opensmile/engine.py
+batchalign/pipelines/segmentation/__init__.py
+batchalign/pipelines/segmentation/cantonese_seg.py
 batchalign/pipelines/speaker/__init__.py
 batchalign/pipelines/speaker/nemo_speaker.py
 batchalign/pipelines/translate/__init__.py

{batchalign-0.8.2.post7 → batchalign-0.8.2.post9}/batchalign.egg-info/requires.txt

@@ -26,6 +26,7 @@ rich-click>=1.7.0
 typing-extensions
 num2words
 tiktoken
+pycantonese
 blobfile
 sentencepiece
 googletrans

{batchalign-0.8.2.post7 → batchalign-0.8.2.post9}/setup.py

@@ -58,6 +58,7 @@ setup(
         "typing-extensions",
         "num2words",
         "tiktoken",
+        "pycantonese",
         "blobfile",
         "sentencepiece",
         "googletrans",

batchalign-0.8.2.post7/batchalign/version

@@ -1,3 +0,0 @@
-0.8.2-post.7
-April 01 2026
-more compare updates