batchalign 0.8.2.post6__tar.gz → 0.8.2.post8__tar.gz

{batchalign-0.8.2.post6/batchalign.egg-info → batchalign-0.8.2.post8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: batchalign
-Version: 0.8.2.post6
+Version: 0.8.2.post8
 Summary: Python Speech Language Sample Analysis
 Author: Brian MacWhinney, Houjun Liu
 Author-email: macw@cmu.edu, houjun@cmu.edu
@@ -42,6 +42,7 @@ Requires-Dist: rich-click>=1.7.0
 Requires-Dist: typing-extensions
 Requires-Dist: num2words
 Requires-Dist: tiktoken
+Requires-Dist: pycantonese
 Requires-Dist: blobfile
 Requires-Dist: sentencepiece
 Requires-Dist: googletrans

{batchalign-0.8.2.post6 → batchalign-0.8.2.post8}/batchalign/cli/cli.py

@@ -348,6 +348,32 @@ def utseg(ctx, in_dir, out_dir, lang, num_speakers, **kwargs):
               in_dir, out_dir,
               loader, writer, C)
 
+#################### SEGMENT ################################
+
+@batchalign.command()
+@common_options
+@click.option("--lang",
+              help="sample language in three-letter ISO 3166-1 alpha-3 code",
+              show_default=True,
+              default="yue",
+              type=str)
+@click.option("--merge-abbrev/--no-merge-abbrev",
+              default=False, help="Merge abbreviations in output. Default: no.")
+@click.pass_context
+def segment(ctx, in_dir, out_dir, lang, **kwargs):
+    """Perform word segmentation on transcripts (Cantonese)."""
+    from batchalign.formats.chat import CHATFile
+
+    def loader(file):
+        return CHATFile(path=os.path.abspath(file)).doc
+
+    def writer(doc, output):
+        CHATFile(doc=doc).write(output, merge_abbrev=kwargs.get("merge_abbrev", False))
+
+    _dispatch("segment", lang, 1, ["cha"], ctx,
+              in_dir, out_dir,
+              loader, writer, C)
+
 #################### BENCHMARK ################################
 
 @batchalign.command()

{batchalign-0.8.2.post6 → batchalign-0.8.2.post8}/batchalign/cli/dispatch.py

@@ -70,6 +70,7 @@ POOL_SAFE_ENGINES = {
     "replacement",
     "ngram",
     "compare_analysis_engine",
+    "cantonese_seg",
 }
 
 warnings.filterwarnings('ignore', category=UserWarning, message='TypedStorage is deprecated')
@@ -234,6 +235,11 @@ def _run_pipeline_for_file(command, pipeline, file, output, loader_info, writer_
         with open(json_path, 'w') as f:
             _json.dump(metrics, f)
 
+    elif command == "segment":
+        doc = CHATFile(path=os.path.abspath(file)).doc
+        doc = pipeline(doc, callback=progress_callback)
+        CHATFile(doc=doc).write(output)
+
     elif command == "opensmile":
         from batchalign.document import Document
         doc = Document.new(media_path=file, lang=local_kwargs.get("lang", kwargs.get("lang", "eng")))
@@ -386,6 +392,7 @@ Cmd2Task = {
     "translate": "translate",
     "opensmile": "opensmile",
     "compare": "compare,compare_analysis",
+    "segment": "segment",
 }
 
 # this is the main runner used by all functions
@@ -406,6 +413,7 @@ def _dispatch(command, lang, num_speakers,
         "benchmark",
         "opensmile",
         "compare",
+        "segment",
     }
     if command in worker_handled:
         # Avoid pickling CLI-local loader/writer functions when the worker

{batchalign-0.8.2.post6 → batchalign-0.8.2.post8}/batchalign/document.py

@@ -27,14 +27,15 @@ class Task(IntEnum):
     DISFLUENCY_ANALYSIS = 6
     RETRACE_ANALYSIS = 7
     UTTERANCE_TIMING_RECOVERY = 8 # "bulletize"
-    FORCED_ALIGNMENT = 9
-    FEATURE_EXTRACT = 10
-    MORPHOSYNTAX = 11
-    COREF = 12
-    WER = 13
-    TRANSLATE = 14
-    COMPARE = 15
-    COMPARE_ANALYSIS = 16
+    WORD_SEGMENTATION = 9
+    FORCED_ALIGNMENT = 10
+    FEATURE_EXTRACT = 11
+    MORPHOSYNTAX = 12
+    COREF = 13
+    WER = 14
+    TRANSLATE = 15
+    COMPARE = 16
+    COMPARE_ANALYSIS = 17
 
 
     DEBUG__G = 0
@@ -51,6 +52,7 @@ TypeMap = {
     Task.SPEAKER_RECOGNITION: TaskType.PROCESSING,
     Task.UTTERANCE_SEGMENTATION: TaskType.PROCESSING,
     Task.UTTERANCE_TIMING_RECOVERY: TaskType.PROCESSING,
+    Task.WORD_SEGMENTATION: TaskType.PROCESSING,
     Task.FORCED_ALIGNMENT: TaskType.PROCESSING,
     Task.MORPHOSYNTAX: TaskType.PROCESSING,
     Task.FEATURE_EXTRACT: TaskType.ANALYSIS,
@@ -73,6 +75,7 @@ TaskFriendlyName = {
     Task.SPEAKER_RECOGNITION: "Speaker Recognition",
     Task.UTTERANCE_SEGMENTATION: "Utterance Segmentation",
     Task.UTTERANCE_TIMING_RECOVERY: "Utterance Timing Recovery",
+    Task.WORD_SEGMENTATION: "Word Segmentation",
     Task.FORCED_ALIGNMENT: "Forced Alignment",
     Task.MORPHOSYNTAX: "Morpho-Syntax",
     Task.FEATURE_EXTRACT: "Feature Extraction",

{batchalign-0.8.2.post6 → batchalign-0.8.2.post8}/batchalign/pipelines/analysis/compare.py

@@ -329,6 +329,10 @@ class CompareEngine(BatchalignEngine):
         # window in the remaining main tokens, then run Levenshtein inside
         # that window to produce the annotations.
         utt_positioned = {i: [] for i in range(len(gold_utterances))}
+        # Track which main (child) forms land in each gold utterance's window
+        utt_main_forms = {i: [] for i in range(len(gold_utterances))}
+        # Track which main utterance(s) contribute to each gold utterance
+        utt_main_speakers = {i: [] for i in range(len(gold_utterances))}
         search_start = 0
 
         for utt_idx in range(len(gold_utterances)):
@@ -345,6 +349,17 @@ class CompareEngine(BatchalignEngine):
             abs_start = search_start + win_start
             abs_end = search_start + win_end
 
+            # Collect unique main forms in the window (deduplicate
+            # across conformed expansions that map to the same form)
+            seen_main = set()
+            for j in range(abs_start, abs_end):
+                orig_idx = main_map[j]
+                if orig_idx not in seen_main:
+                    seen_main.add(orig_idx)
+                    m_utt_idx, m_form_idx, m_form = main_info[orig_idx]
+                    utt_main_forms[utt_idx].append(m_form)
+                    utt_main_speakers[utt_idx].append(m_utt_idx)
+
             # Align the chosen window against this gold utterance
             window_main = conformed_main[abs_start:abs_end]
             utt_alignment = align(window_main, g_tokens, False, match_fn)
@@ -363,11 +378,6 @@ class CompareEngine(BatchalignEngine):
                     gold_form = gold_info[orig_gold_idx][2]
                     last_gold_form_idx = gold_form_idx
 
-                    # Copy timing from main; keep gold's own morphology
-                    if main_form.time is not None:
-                        gold_form.time = main_form.time
-                    gold_form.dependency = None
-
                     utt_positioned[utt_idx].append((gold_form_idx, CompareToken(
                         text=item.key,
                         pos=_get_pos(gold_form),
@@ -383,8 +393,6 @@ class CompareEngine(BatchalignEngine):
                         gold_form = gold_info[orig_gold_idx][2]
                         last_gold_form_idx = gold_form_idx
 
-                        gold_form.dependency = None
-
                         utt_positioned[utt_idx].append((gold_form_idx, CompareToken(
                             text=item.key,
                             pos=_get_pos(gold_form),
@@ -415,21 +423,41 @@ class CompareEngine(BatchalignEngine):
                     pos="PUNCT",
                     status="match"
                 )))
-                # Ensure punctuation forms have morphology so %mor includes
-                # the final delimiter (matches generator's PUNCT|<char> logic)
-                if form.morphology is None:
-                    form.morphology = [Morphology(
-                        pos="PUNCT", lemma=form.text.strip(), feats=""
-                    )]
-                form.dependency = None
             utt_positioned[utt_idx].sort(key=lambda x: x[0])
 
-        # --- 7. Set comparison on each gold utterance ---
+        # --- 7. Replace each gold utterance's main line with the child's
+        #         (main/proposal) forms and set the child's speaker. ---
         for utt_idx, utt in enumerate(gold_utterances):
             tokens = [tok for _, tok in utt_positioned[utt_idx]]
             utt.comparison = tokens if tokens else None
 
-            timed_forms = [form for form in utt.content if form.time is not None]
+            # Replace content with the child/main forms from the window
+            child_forms = utt_main_forms[utt_idx]
+            if child_forms:
+                # Grab the ending punctuation from gold so the utterance
+                # delimiter is preserved
+                gold_ending = [
+                    f for f in utt.content
+                    if f.text.strip() in ENDING_PUNCT
+                ]
+                # Strip dependency from child forms (morphosyntax was run
+                # on the main doc separately; gold structure differs)
+                for f in child_forms:
+                    f.dependency = None
+                utt.content = child_forms + gold_ending
+
+                # Set speaker from the child utterance that contributed
+                # the most forms to this window
+                speaker_indices = utt_main_speakers[utt_idx]
+                if speaker_indices:
+                    # Pick the most common main utterance index
+                    majority_utt_idx = max(
+                        set(speaker_indices), key=speaker_indices.count
+                    )
+                    utt.tier = main_utterances[majority_utt_idx].tier
+
+            # Derive utterance timing from the (now-child) forms
+            timed_forms = [f for f in utt.content if f.time is not None]
             if timed_forms:
                 utt.time = (timed_forms[0].time[0], timed_forms[-1].time[1])
                 utt.text = None

{batchalign-0.8.2.post6 → batchalign-0.8.2.post8}/batchalign/pipelines/dispatch.py

@@ -16,6 +16,7 @@ DEFAULT_PACKAGES = {
     "asr": "whisper_oai",
     "utr": "whisper_utr",
     "fa": "whisper_fa",
+    "segment": "cantonese_seg",
     "speaker": "pyannote",
     "morphosyntax": "stanza",
     "disfluency": "replacement",
@@ -56,6 +57,8 @@ def resolve_engine_specs(pkg_str, lang, num_speakers=None, **arg_overrides):
     if "fa" in packages:
         if "utr" not in packages:
             packages.append("utr")
+        if "segment" not in packages and lang == "yue":
+            packages.append("segment")
 
     overrides = LANGUAGE_OVERRIDE_PACKAGES.get(lang, {})
     specs = []
@@ -123,6 +126,9 @@ def dispatch_pipeline(pkg_str, lang, num_speakers=None, **arg_overrides):
         elif engine == "ngram":
             from batchalign.pipelines.cleanup import NgramRetraceEngine
             engines.append(NgramRetraceEngine())
+        elif engine == "cantonese_seg":
+            from batchalign.pipelines.segmentation import CantoneseSegmentationEngine
+            engines.append(CantoneseSegmentationEngine())
         elif engine == "whisper_fa":
             from batchalign.pipelines.fa import WhisperFAEngine
             engines.append(WhisperFAEngine())

batchalign-0.8.2.post8/batchalign/pipelines/segmentation/__init__.py

@@ -0,0 +1,15 @@
+"""
+Segmentation Pipeline Module
+
+Provides engines for word segmentation of unsegmented text.
+Uses lazy loading to avoid importing heavy dependencies until needed.
+"""
+
+def __getattr__(name):
+    """Lazy loading for segmentation engines."""
+    if name == 'CantoneseSegmentationEngine':
+        from .cantonese_seg import CantoneseSegmentationEngine
+        return CantoneseSegmentationEngine
+    raise AttributeError(f"module '{__name__}' has no attribute '{name}'")
+
+__all__ = ['CantoneseSegmentationEngine']

batchalign-0.8.2.post8/batchalign/pipelines/segmentation/cantonese_seg.py

@@ -0,0 +1,109 @@
+from batchalign.document import *
+from batchalign.pipelines.base import *
+from batchalign.utils.dp import *
+from batchalign.constants import *
+
+import logging
+L = logging.getLogger("batchalign")
+
+
+class CantoneseSegmentationEngine(BatchalignEngine):
+    tasks = [Task.WORD_SEGMENTATION]
+
+    def __init__(self):
+        import pycantonese
+        self.__segment = pycantonese.segment
+
+    def process(self, doc: Document, **kwargs):
+        # no-op for non-Cantonese
+        if "yue" not in doc.langs:
+            L.debug("CantoneseSegmentation: skipping non-Cantonese document.")
+            return doc
+
+        for ut in doc.content:
+            if not isinstance(ut, Utterance):
+                continue
+
+            # separate regular words from punctuation/features at the end
+            regular_forms = []
+            trailing = []
+            for f in ut.content:
+                if f.type in (TokenType.PUNCT, TokenType.FEAT, TokenType.ANNOT):
+                    trailing.append(f)
+                else:
+                    # if we already started collecting trailing punct,
+                    # but hit another regular token, flush trailing back
+                    if trailing:
+                        regular_forms.extend(trailing)
+                        trailing = []
+                    regular_forms.append(f)
+
+            if not regular_forms:
+                continue
+
+            # combine all regular token text (no spaces for CJK)
+            original_text = "".join(f.text for f in regular_forms)
+            if not original_text.strip():
+                continue
+
+            # run pycantonese segmentation
+            segmented_words = self.__segment(original_text)
+
+            # use character-level DP alignment to map segmented words
+            # back onto original tokens
+            #
+            # reference: chars from original tokens, payload = index into regular_forms
+            ref_targets = []
+            for i, f in enumerate(regular_forms):
+                for ch in f.text:
+                    ref_targets.append(ReferenceTarget(ch, payload=i))
+
+            # payload: chars from segmented words, payload = index into segmented_words
+            pay_targets = []
+            for i, word in enumerate(segmented_words):
+                for ch in word:
+                    pay_targets.append(PayloadTarget(ch, payload=i))
+
+            alignments = align(pay_targets, ref_targets, tqdm=False)
+
+            # build a mapping: segmented word index -> list of original token indices
+            seg_to_orig: dict[int, list[int]] = {}
+            for elem in alignments:
+                if isinstance(elem, Match):
+                    seg_to_orig.setdefault(elem.payload, []).append(elem.reference_payload)
+
+            # construct new Form objects from segmented words
+            new_forms = []
+            for seg_idx, word in enumerate(segmented_words):
+                orig_indices = seg_to_orig.get(seg_idx, [])
+
+                # inherit timing: start from first orig, end from last orig
+                time = None
+                if orig_indices:
+                    first_time = regular_forms[orig_indices[0]].time
+                    last_time = regular_forms[orig_indices[-1]].time
+                    if first_time is not None and last_time is not None:
+                        time = (first_time[0], last_time[1])
+                    elif first_time is not None:
+                        time = first_time
+                    elif last_time is not None:
+                        time = last_time
+
+                # determine token type from the first matched original
+                token_type = TokenType.REGULAR
+                if orig_indices:
+                    token_type = regular_forms[orig_indices[0]].type
+
+                new_forms.append(Form(
+                    text=word,
+                    time=time,
+                    type=token_type,
+                ))
+
+            # re-append trailing punctuation/features
+            new_forms.extend(trailing)
+            ut.content = new_forms
+            # clear cached text so _detokenize() regenerates from new Forms
+            ut.text = None
+
+        return doc

batchalign-0.8.2.post8/batchalign/version

@@ -0,0 +1,3 @@
+0.8.2-post.8
+April 02 2026
+segment

{batchalign-0.8.2.post6 → batchalign-0.8.2.post8/batchalign.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: batchalign
-Version: 0.8.2.post6
+Version: 0.8.2.post8
 Summary: Python Speech Language Sample Analysis
 Author: Brian MacWhinney, Houjun Liu
 Author-email: macw@cmu.edu, houjun@cmu.edu
@@ -42,6 +42,7 @@ Requires-Dist: rich-click>=1.7.0
 Requires-Dist: typing-extensions
 Requires-Dist: num2words
 Requires-Dist: tiktoken
+Requires-Dist: pycantonese
 Requires-Dist: blobfile
 Requires-Dist: sentencepiece
 Requires-Dist: googletrans

{batchalign-0.8.2.post6 → batchalign-0.8.2.post8}/batchalign.egg-info/SOURCES.txt

@@ -107,6 +107,8 @@ batchalign/pipelines/morphosyntax/fr/case.py
 batchalign/pipelines/morphosyntax/ja/verbforms.py
 batchalign/pipelines/opensmile/__init__.py
 batchalign/pipelines/opensmile/engine.py
+batchalign/pipelines/segmentation/__init__.py
+batchalign/pipelines/segmentation/cantonese_seg.py
 batchalign/pipelines/speaker/__init__.py
 batchalign/pipelines/speaker/nemo_speaker.py
 batchalign/pipelines/translate/__init__.py

{batchalign-0.8.2.post6 → batchalign-0.8.2.post8}/batchalign.egg-info/requires.txt

@@ -26,6 +26,7 @@ rich-click>=1.7.0
 typing-extensions
 num2words
 tiktoken
+pycantonese
 blobfile
 sentencepiece
 googletrans

{batchalign-0.8.2.post6 → batchalign-0.8.2.post8}/setup.py

@@ -58,6 +58,7 @@ setup(
         "typing-extensions",
         "num2words",
         "tiktoken",
+        "pycantonese",
         "blobfile",
         "sentencepiece",
         "googletrans",

batchalign-0.8.2.post6/batchalign/version

@@ -1,3 +0,0 @@
-0.8.2-post.6
-March 28 2026
-fix bugs