bambara-text-normalizer 1.0.0__py3-none-any.whl

bambara_normalizer/__init__.py

@@ -0,0 +1,187 @@
+# Copyright 2026 sudoping01.
+
+# Licensed under the MIT License; you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+
+# https://opensource.org/licenses/MIT
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__version__ = "1.0.0"
+__author__ = "sudoping01"
+__license__ = "MIT"
+
+
+from .dates import (
+    DAYS_OF_WEEK,
+    MONTHS,
+    bambara_to_date,
+    bambara_to_day_of_week,
+    bambara_to_month,
+    date_to_bambara,
+    day_of_week_to_bambara,
+    denormalize_dates_in_text,
+    format_date_bambara,
+    is_bambara_day,
+    is_bambara_month,
+    month_to_bambara,
+    normalize_dates_in_text,
+)
+from .evaluation import (
+    BambaraEvaluator,
+    BambaraTransform,
+    EvaluationResult,
+    compute_cer,
+    compute_der,
+    compute_mer,
+    compute_wer,
+    compute_wil,
+    compute_wip,
+    create_bambara_transform,
+    evaluate,
+    evaluate_batch,
+    visualize_alignment,
+)
+from .normalizer import (
+    BambaraNormalizer,
+    BambaraNormalizerConfig,
+    create_normalizer,
+)
+from .numbers import (
+    bambara_to_number,
+    denormalize_numbers_in_text,
+    is_number_word,
+    normalize_numbers_in_text,
+    number_to_bambara,
+    number_to_ordinal,
+)
+from .times import (
+    LERE,
+    MINITI,
+    NEGE_KANYE,
+    SEGONI,
+    bambara_to_duration,
+    bambara_to_time,
+    denormalize_times_in_text,
+    duration_to_bambara,
+    format_duration_bambara,
+    format_time_bambara,
+    is_time_word,
+    normalize_times_in_text,
+    time_to_bambara,
+)
+from .utils import (
+    BAMBARA_ALPHABET,
+    BAMBARA_CONSONANTS,
+    BAMBARA_SPECIAL_CHARS,
+    BAMBARA_VOWELS,
+    add_tone,
+    analyze_text,
+    count_tone_marks,
+    get_base_char,
+    get_tone,
+    get_unicode_info,
+    has_tone_marks,
+    is_bambara_char,
+    is_bambara_special_char,
+    normalize_unicode_variants,
+    remove_tones,
+    validate_bambara_text,
+)
+
+__all__ = [
+    "BambaraNormalizer",
+    "BambaraNormalizerConfig",
+    "NormalizationLevel",
+    "BambaraEvaluator",
+    "EvaluationResult",
+    "create_normalizer",
+    "compute_wer",
+    "compute_cer",
+    "compute_der",
+    "compute_mer",
+    "compute_wil",
+    "compute_wip",
+    "evaluate",
+    "evaluate_batch",
+    "visualize_alignment",
+    "create_bambara_transform",
+    "BambaraTransform",
+    "is_bambara_char",
+    "is_bambara_special_char",
+    "get_base_char",
+    "get_tone",
+    "add_tone",
+    "remove_tones",
+    "has_tone_marks",
+    "count_tone_marks",
+    "validate_bambara_text",
+    "analyze_text",
+    "normalize_unicode_variants",
+    "get_unicode_info",
+    "BAMBARA_VOWELS",
+    "BAMBARA_CONSONANTS",
+    "BAMBARA_SPECIAL_CHARS",
+    "BAMBARA_ALPHABET",
+    "number_to_bambara",
+    "bambara_to_number",
+    "normalize_numbers_in_text",
+    "denormalize_numbers_in_text",
+    "is_number_word",
+    "number_to_ordinal",
+    "date_to_bambara",
+    "bambara_to_date",
+    "format_date_bambara",
+    "day_of_week_to_bambara",
+    "bambara_to_day_of_week",
+    "month_to_bambara",
+    "bambara_to_month",
+    "normalize_dates_in_text",
+    "denormalize_dates_in_text",
+    "is_bambara_month",
+    "is_bambara_day",
+    "DAYS_OF_WEEK",
+    "MONTHS",
+    "time_to_bambara",
+    "bambara_to_time",
+    "format_time_bambara",
+    "duration_to_bambara",
+    "bambara_to_duration",
+    "format_duration_bambara",
+    "normalize_times_in_text",
+    "denormalize_times_in_text",
+    "is_time_word",
+    "NEGE_KANYE",
+    "LERE",
+    "MINITI",
+    "SEGONI",
+]
+
+
+def normalize(text: str, preset: str = "standard", **kwargs) -> str:
+    """Convenience function for quick normalization.
+
+    Args:
+        text: Text to normalize
+        preset: Preset configuration ("standard", "wer", "cer", "minimal")
+        **kwargs: Override specific configuration options
+
+    Returns:
+        Normalized text
+
+    Example:
+        >>> from bambara_normalizer import normalize
+        >>> normalize("B'a fɔ́!")
+        'bɛ a fɔ'
+        >>> normalize("B'a fɔ́!", preset="wer")
+        'bɛ a fɔ'
+    """
+    normalizer = create_normalizer(preset, **kwargs)
+    return normalizer(text)
+
+
+__all__.append("normalize")

bambara_normalizer/cli.py

@@ -0,0 +1,298 @@
+# Copyright 2026 sudoping01.
+
+# Licensed under the MIT License; you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+
+# https://opensource.org/licenses/MIT
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+"""
+Command-line interface for Bambara text normalizer.
+
+Usage:
+    bambara-normalize "B'a fɔ́"
+    bambara-normalize --mode expand "B'a fɔ́"
+    bambara-normalize --mode contract "bɛ a fɔ"
+    bambara-normalize --preset wer --mode contract "text"
+    echo "text" | bambara-normalize
+    bambara-normalize --file input.txt --output output.txt
+    bambara-normalize --evaluate ref.txt hyp.txt
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+from .evaluation import BambaraEvaluator
+from .normalizer import BambaraNormalizer, BambaraNormalizerConfig, create_normalizer
+
+
+def parse_args(args: list[str] | None = None) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        prog="bambara-normalize",
+        description="Bambara text normalizer for ASR evaluation",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  Normalize text (expand contractions, default):
+    bambara-normalize "B'a fɔ́"
+    bambara-normalize --mode expand "k'a ta"
+
+  Contract expanded forms:
+    bambara-normalize --mode contract "bɛ a fɔ"
+    bambara-normalize --mode contract "ka a ta"
+
+  Preserve contractions (don't touch):
+    bambara-normalize --mode preserve "B'a fɔ"
+
+  Use WER preset with contraction:
+    bambara-normalize --preset wer --mode contract "ka a ta"
+
+  Process file:
+    bambara-normalize --file input.txt --output normalized.txt
+    bambara-normalize --mode contract --file input.txt
+
+  Evaluate ASR output:
+    bambara-normalize --evaluate reference.txt hypothesis.txt
+    bambara-normalize --evaluate --mode contract ref.txt hyp.txt
+
+  Pipe from stdin:
+    echo "B'a fɔ́" | bambara-normalize
+    echo "bɛ a fɔ" | bambara-normalize --mode contract
+""",
+    )
+
+    parser.add_argument(
+        "text",
+        nargs="?",
+        help="Text to normalize (reads from stdin if not provided)",
+    )
+
+    parser.add_argument(
+        "--mode",
+        "-m",
+        choices=["expand", "contract", "preserve"],
+        default="expand",
+        help="Contraction mode: expand (default), contract, or preserve",
+    )
+
+    parser.add_argument(
+        "--preset",
+        "-p",
+        choices=["standard", "wer", "cer", "minimal", "preserving_tones"],
+        default="standard",
+        help="Normalization preset (default: standard)",
+    )
+
+    parser.add_argument(
+        "--file",
+        "-f",
+        type=Path,
+        help="Input file to normalize",
+    )
+
+    parser.add_argument(
+        "--output",
+        "-o",
+        type=Path,
+        help="Output file (default: stdout)",
+    )
+
+    parser.add_argument(
+        "--evaluate",
+        "-e",
+        nargs=2,
+        metavar=("REF", "HYP"),
+        type=Path,
+        help="Evaluate hypothesis against reference (provide two files)",
+    )
+
+    parser.add_argument(
+        "--preserve-tones",
+        action="store_true",
+        help="Preserve tone marks during normalization",
+    )
+
+    parser.add_argument(
+        "--expand-numbers",
+        action="store_true",
+        help="Expand numbers to Bambara words",
+    )
+
+    parser.add_argument(
+        "--debug",
+        action="store_true",
+        help="Show normalization steps",
+    )
+
+    parser.add_argument(
+        "--version",
+        "-v",
+        action="version",
+        version="%(prog)s 2.0.0",
+    )
+
+    return parser.parse_args(args)
+
+
+def normalize_text(
+    text: str,
+    preset: str,
+    mode: str = "expand",
+    preserve_tones: bool = False,
+    expand_numbers: bool = False,
+    debug: bool = False,
+) -> str:
+    kwargs = {}
+    if preserve_tones:
+        kwargs["preserve_tones"] = True
+    if expand_numbers:
+        kwargs["expand_numbers"] = True
+
+    normalizer = create_normalizer(preset, mode=mode, **kwargs)
+
+    if debug:
+        diff = normalizer.get_normalization_diff(text)
+        print("Normalization steps:", file=sys.stderr)
+        for step, value in diff.items():
+            print(f"  {step}: {value}", file=sys.stderr)
+        print(file=sys.stderr)
+
+    return normalizer(text)
+
+
+def process_file(
+    input_path: Path,
+    output_path: Path | None,
+    normalizer: BambaraNormalizer,
+) -> None:
+    with open(input_path, encoding="utf-8") as f:
+        lines = f.readlines()
+
+    normalized = [normalizer(line.rstrip("\n")) for line in lines]
+
+    if output_path:
+        with open(output_path, "w", encoding="utf-8") as f:
+            f.write("\n".join(normalized) + "\n")
+        print(f"Normalized output written to {output_path}", file=sys.stderr)
+    else:
+        for line in normalized:
+            print(line)
+
+
+def run_evaluation(
+    ref_path: Path,
+    hyp_path: Path,
+    preset: str,
+    mode: str = "expand",
+) -> None:
+    with open(ref_path, encoding="utf-8") as f:
+        references = [line.strip() for line in f if line.strip()]
+
+    with open(hyp_path, encoding="utf-8") as f:
+        hypotheses = [line.strip() for line in f if line.strip()]
+
+    if len(references) != len(hypotheses):
+        print(
+            f"Error: Reference ({len(references)} lines) and hypothesis "
+            f"({len(hypotheses)} lines) have different lengths",
+            file=sys.stderr,
+        )
+        sys.exit(1)
+
+    config = BambaraNormalizerConfig.for_wer_evaluation(mode=mode)
+    evaluator = BambaraEvaluator(config=config)
+    aggregate, individual = evaluator.evaluate_batch(references, hypotheses)
+
+    print("=" * 60)
+    print("Bambara ASR Evaluation Results")
+    print("=" * 60)
+    print(f"Reference file: {ref_path}")
+    print(f"Hypothesis file: {hyp_path}")
+    print(f"Normalization preset: {preset}")
+    print(f"Contraction mode: {mode}")
+    print(f"Total utterances: {len(references)}")
+    print("-" * 60)
+    print(f"Word Error Rate (WER): {aggregate.wer:.2%}")
+    print(f"  Substitutions: {aggregate.word_substitutions}")
+    print(f"  Deletions: {aggregate.word_deletions}")
+    print(f"  Insertions: {aggregate.word_insertions}")
+    print(f"  Reference words: {aggregate.total_reference_words}")
+    print("-" * 60)
+    print(f"Character Error Rate (CER): {aggregate.cer:.2%}")
+    print(f"  Substitutions: {aggregate.char_substitutions}")
+    print(f"  Deletions: {aggregate.char_deletions}")
+    print(f"  Insertions: {aggregate.char_insertions}")
+    print(f"  Reference chars: {aggregate.total_reference_chars}")
+    print("=" * 60)
+
+
+def main(args: list[str] | None = None) -> int:
+    parsed = parse_args(args)
+
+    try:
+        if parsed.evaluate:
+            ref_path, hyp_path = parsed.evaluate
+            run_evaluation(ref_path, hyp_path, parsed.preset, parsed.mode)
+            return 0
+
+        kwargs = {}
+        if parsed.preserve_tones:
+            kwargs["preserve_tones"] = True
+        if parsed.expand_numbers:
+            kwargs["expand_numbers"] = True
+
+        normalizer = create_normalizer(parsed.preset, mode=parsed.mode, **kwargs)
+
+        if parsed.file:
+            process_file(parsed.file, parsed.output, normalizer)
+            return 0
+
+        if parsed.text:
+            text = parsed.text
+        elif not sys.stdin.isatty():
+            text = sys.stdin.read().strip()
+        else:
+            print("Error: No input provided", file=sys.stderr)
+            print(
+                "Usage: bambara-normalize <text> or echo <text> | bambara-normalize",
+                file=sys.stderr,
+            )
+            return 1
+
+        result = normalize_text(
+            text,
+            parsed.preset,
+            parsed.mode,
+            parsed.preserve_tones,
+            parsed.expand_numbers,
+            parsed.debug,
+        )
+
+        if parsed.output:
+            with open(parsed.output, "w", encoding="utf-8") as f:
+                f.write(result + "\n")
+            print(f"Output written to {parsed.output}", file=sys.stderr)
+        else:
+            print(result)
+
+        return 0
+
+    except FileNotFoundError as e:
+        print(f"Error: File not found: {e}", file=sys.stderr)
+        return 1
+    except Exception as e:
+        print(f"Error: {e}", file=sys.stderr)
+        return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

bambara_normalizer/config.py

@@ -0,0 +1,197 @@
+# Copyright 2026 sudoping01.
+
+# Licensed under the MIT License; you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+
+# https://opensource.org/licenses/MIT
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import Enum
+
+
+class ContractionMode(Enum):
+    EXPAND = "expand"
+    CONTRACT = "contract"
+    PRESERVE = "preserve"
+
+
+class NormalizationLevel(Enum):
+    MINIMAL = "minimal"
+    STANDARD = "standard"
+    AGGRESSIVE = "aggressive"
+
+
+@dataclass
+class BambaraNormalizerConfig:
+    """
+    Configuration for Bambara text normalization.
+
+    Attributes:
+        contraction_mode: How to handle contractions ("expand", "contract", "preserve")
+        preserve_tones: Keep tone diacritics (à, á, etc.)
+        normalize_legacy_orthography: Convert old spellings (è→ɛ, ny→ɲ)
+        lowercase: Convert to lowercase
+        remove_punctuation: Remove punctuation marks
+        normalize_whitespace: Collapse multiple spaces
+        normalize_apostrophes: Standardize apostrophe variants
+        normalize_special_chars: Handle variant forms of ɛ, ɔ, ɲ, ŋ
+        expand_numbers: Convert digits to Bambara words
+        expand_dates: Convert date patterns (DD-MM-YYYY) to Bambara
+        remove_diacritics_except_tones: Remove non-tonal diacritics
+        handle_french_loanwords: Apply French word normalization
+        strip_repetitions: Normalize repeated characters
+        normalize_compounds: Standardize compound word spacing
+        expand_times: Convert time patterns to Bambara
+    """
+
+    contraction_mode: str = "expand"
+    preserve_tones: bool = True
+    normalize_legacy_orthography: bool = True
+    lowercase: bool = True
+    remove_punctuation: bool = True
+    normalize_whitespace: bool = True
+    normalize_apostrophes: bool = True
+    normalize_special_chars: bool = True
+    expand_numbers: bool = False
+    expand_dates: bool = False
+    expand_times: bool = False
+    remove_diacritics_except_tones: bool = False
+    handle_french_loanwords: bool = True
+    strip_repetitions: bool = False
+    normalize_compounds: bool = False
+
+    @property
+    def expand_contractions(self) -> bool:
+        """Backward compatibility: True if mode is 'expand'."""
+        return self.contraction_mode == "expand"
+
+    @expand_contractions.setter
+    def expand_contractions(self, value: bool) -> None:
+        """Backward compatibility setter."""
+        if value:
+            self.contraction_mode = "expand"
+        else:
+            self.contraction_mode = "preserve"
+
+    @classmethod
+    def for_wer_evaluation(cls, mode: str = "expand") -> BambaraNormalizerConfig:
+        return cls(
+            contraction_mode=mode,
+            preserve_tones=False,
+            normalize_legacy_orthography=True,
+            lowercase=True,
+            remove_punctuation=True,
+            normalize_whitespace=True,
+            normalize_apostrophes=True,
+            normalize_special_chars=True,
+            expand_numbers=True,
+            expand_dates=True,
+            expand_times=True,
+            remove_diacritics_except_tones=True,
+            handle_french_loanwords=True,
+            strip_repetitions=True,
+            normalize_compounds=True,
+        )
+
+    @classmethod
+    def for_cer_evaluation(cls, mode: str = "expand") -> BambaraNormalizerConfig:
+        return cls(
+            contraction_mode=mode,
+            preserve_tones=False,
+            normalize_legacy_orthography=True,
+            lowercase=True,
+            remove_punctuation=True,
+            normalize_whitespace=True,
+            normalize_apostrophes=True,
+            normalize_special_chars=True,
+            expand_numbers=True,
+            expand_dates=True,
+            expand_times=True,
+            remove_diacritics_except_tones=True,
+            handle_french_loanwords=True,
+            strip_repetitions=False,
+            normalize_compounds=True,
+        )
+
+    @classmethod
+    def preserving_tones(cls, mode: str = "expand") -> BambaraNormalizerConfig:
+        return cls(
+            contraction_mode=mode,
+            preserve_tones=True,
+            normalize_legacy_orthography=True,
+            lowercase=True,
+            remove_punctuation=True,
+            normalize_whitespace=True,
+            normalize_apostrophes=True,
+            normalize_special_chars=True,
+            expand_numbers=False,
+            expand_dates=False,
+            expand_times=False,
+            remove_diacritics_except_tones=False,
+            handle_french_loanwords=False,
+            strip_repetitions=False,
+            normalize_compounds=False,
+        )
+
+    @classmethod
+    def minimal(cls) -> BambaraNormalizerConfig:
+        return cls(
+            contraction_mode="preserve",
+            preserve_tones=True,
+            normalize_legacy_orthography=False,
+            lowercase=True,
+            remove_punctuation=False,
+            normalize_whitespace=True,
+            normalize_apostrophes=True,
+            normalize_special_chars=False,
+            expand_numbers=False,
+            expand_dates=False,
+            expand_times=False,
+            remove_diacritics_except_tones=False,
+            handle_french_loanwords=False,
+            strip_repetitions=False,
+            normalize_compounds=False,
+        )
+
+
+@dataclass
+class EvaluationResult:
+    wer: float
+    cer: float
+    mer: float = 0.0
+    wil: float = 0.0
+    wip: float = 0.0
+    der: float | None = None
+
+    word_substitutions: int = 0
+    word_deletions: int = 0
+    word_insertions: int = 0
+    word_hits: int = 0
+    total_reference_words: int = 0
+
+    char_substitutions: int = 0
+    char_deletions: int = 0
+    char_insertions: int = 0
+    char_hits: int = 0
+    total_reference_chars: int = 0
+
+    reference_normalized: str = ""
+    hypothesis_normalized: str = ""
+
+    def __str__(self) -> str:
+        return (
+            f"WER: {self.wer:.2%} "
+            f"(S={self.word_substitutions}, D={self.word_deletions}, I={self.word_insertions}, H={self.word_hits}, N={self.total_reference_words})\n"
+            f"CER: {self.cer:.2%} "
+            f"(S={self.char_substitutions}, D={self.char_deletions}, I={self.char_insertions}, H={self.char_hits}, N={self.total_reference_chars})\n"
+            f"MER: {self.mer:.2%} | WIL: {self.wil:.2%} | WIP: {self.wip:.2%}"
+            + (f"\nDER: {self.der:.2%}" if self.der is not None else "")
+        )