OpenSTBench 1.1.0__tar.gz → 1.2.2__tar.gz

{openstbench-1.1.0/src/OpenSTBench.egg-info → openstbench-1.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: OpenSTBench
-Version: 1.1.0
+Version: 1.2.2
 Summary: Unified multidimensional evaluation toolkit for S2TT and S2ST systems in offline and streaming speech translation settings
 Author-email: Yanjie An <691476922@qq.com>
 License: MIT
@@ -17,6 +17,7 @@ Classifier: Programming Language :: Python :: 3
 Requires-Python: <3.11,>=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: setuptools==80.9.0
 Requires-Dist: torch<2.7,>=2.6
 Requires-Dist: torchaudio<2.7,>=2.6
 Requires-Dist: numpy
@@ -36,6 +37,7 @@ Provides-Extra: whisper
 Requires-Dist: openai-whisper; extra == "whisper"
 Provides-Extra: speech-quality
 Requires-Dist: jiwer; extra == "speech-quality"
+Requires-Dist: opencc-python-reimplemented; extra == "speech-quality"
 Provides-Extra: emotion
 Requires-Dist: funasr; extra == "emotion"
 Requires-Dist: modelscope; extra == "emotion"
@@ -45,6 +47,7 @@ Provides-Extra: all
 Requires-Dist: unbabel-comet>=2.2.2; extra == "all"
 Requires-Dist: openai-whisper; extra == "all"
 Requires-Dist: jiwer; extra == "all"
+Requires-Dist: opencc-python-reimplemented; extra == "all"
 Requires-Dist: funasr; extra == "all"
 Requires-Dist: modelscope; extra == "all"
 Requires-Dist: transformers; extra == "all"

{openstbench-1.1.0 → openstbench-1.2.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "OpenSTBench"
-version = "1.1.0"  
+version = "1.2.2"  
 description = "Unified multidimensional evaluation toolkit for S2TT and S2ST systems in offline and streaming speech translation settings"
 readme = {file = "README.md", content-type = "text/markdown"}
 requires-python = ">=3.9,<3.11"
@@ -29,6 +29,7 @@ classifiers = [
 ]
 
 dependencies = [
+    "setuptools==80.9.0",
     "torch>=2.6,<2.7",
     "torchaudio>=2.6,<2.7",
     "numpy",
@@ -53,10 +54,10 @@ Documentation = "https://github.com/sjtuayj/OpenSTBench#readme"
 [project.optional-dependencies]
 comet = ["unbabel-comet>=2.2.2"]
 whisper = ["openai-whisper"]
-speech_quality = ["jiwer"]
+speech_quality = ["jiwer", "opencc-python-reimplemented"]
 emotion = ["funasr", "modelscope"]  
 paralinguistics = ["transformers"]
-all = ["unbabel-comet>=2.2.2", "openai-whisper", "jiwer", "funasr", "modelscope", "transformers"]
+all = ["unbabel-comet>=2.2.2", "openai-whisper", "jiwer", "opencc-python-reimplemented", "funasr", "modelscope", "transformers"]
 
 [tool.setuptools]
 package-dir = {"" = "src"}

{openstbench-1.1.0 → openstbench-1.2.2/src/OpenSTBench.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: OpenSTBench
-Version: 1.1.0
+Version: 1.2.2
 Summary: Unified multidimensional evaluation toolkit for S2TT and S2ST systems in offline and streaming speech translation settings
 Author-email: Yanjie An <691476922@qq.com>
 License: MIT
@@ -17,6 +17,7 @@ Classifier: Programming Language :: Python :: 3
 Requires-Python: <3.11,>=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: setuptools==80.9.0
 Requires-Dist: torch<2.7,>=2.6
 Requires-Dist: torchaudio<2.7,>=2.6
 Requires-Dist: numpy
@@ -36,6 +37,7 @@ Provides-Extra: whisper
 Requires-Dist: openai-whisper; extra == "whisper"
 Provides-Extra: speech-quality
 Requires-Dist: jiwer; extra == "speech-quality"
+Requires-Dist: opencc-python-reimplemented; extra == "speech-quality"
 Provides-Extra: emotion
 Requires-Dist: funasr; extra == "emotion"
 Requires-Dist: modelscope; extra == "emotion"
@@ -45,6 +47,7 @@ Provides-Extra: all
 Requires-Dist: unbabel-comet>=2.2.2; extra == "all"
 Requires-Dist: openai-whisper; extra == "all"
 Requires-Dist: jiwer; extra == "all"
+Requires-Dist: opencc-python-reimplemented; extra == "all"
 Requires-Dist: funasr; extra == "all"
 Requires-Dist: modelscope; extra == "all"
 Requires-Dist: transformers; extra == "all"

{openstbench-1.1.0 → openstbench-1.2.2}/src/OpenSTBench.egg-info/SOURCES.txt

@@ -9,6 +9,7 @@ src/OpenSTBench.egg-info/top_level.txt
 src/openstbench/__init__.py
 src/openstbench/_model_loading.py
 src/openstbench/emotion_evaluator.py
+src/openstbench/language_policy.py
 src/openstbench/paralinguistic_evaluator.py
 src/openstbench/speaker_similarity_evaluator.py
 src/openstbench/speech_quality_evaluator.py

{openstbench-1.1.0 → openstbench-1.2.2}/src/OpenSTBench.egg-info/requires.txt

@@ -1,3 +1,4 @@
+setuptools==80.9.0
 torch<2.7,>=2.6
 torchaudio<2.7,>=2.6
 numpy
@@ -16,6 +17,7 @@ resemblyzer
 unbabel-comet>=2.2.2
 openai-whisper
 jiwer
+opencc-python-reimplemented
 funasr
 modelscope
 transformers
@@ -32,6 +34,7 @@ transformers
 
 [speech_quality]
 jiwer
+opencc-python-reimplemented
 
 [whisper]
 openai-whisper

{openstbench-1.1.0 → openstbench-1.2.2}/src/openstbench/__init__.py

@@ -9,7 +9,7 @@ actually accessed.
 from importlib import import_module
 from typing import Dict, Tuple
 
-__version__ = "1.1.0"
+__version__ = "1.2.2"
 
 __all__ = [
     "TranslationEvaluator",
@@ -35,6 +35,8 @@ __all__ = [
     "AgentPipeline",
     "ReadAction",
     "WriteAction",
+    "TextSegment",
+    "SpeechSegment",
     "load_text_from_file_or_list",
     "load_audio_from_folder",
     # "Dataset",
@@ -87,7 +89,7 @@ _EXPORT_SPECS: Tuple[Tuple[str, Tuple[str, ...]], ...] = (
     #     ),
     # ),
     ("latency.agent", ("AgentPipeline", "GenericAgent")),
-    ("latency.basics", ("ReadAction", "WriteAction")),
+    ("latency.basics", ("ReadAction", "WriteAction", "TextSegment", "SpeechSegment")),
     ("latency.cli", ("LatencyEvaluator",)),
 )
 

openstbench-1.2.2/src/openstbench/language_policy.py

@@ -0,0 +1,91 @@
+from __future__ import annotations
+
+from typing import Optional
+
+
+LANGUAGE_POLICY_VERSION = "2026-07-fleurs-wer-cer-v1"
+
+_LANGUAGE_ALIASES = {
+    "cmn": "zh",
+    "zho": "zh",
+    "chi": "zh",
+    "zh": "zh",
+    "yue": "zh",
+    "eng": "en",
+    "en": "en",
+    "fra": "fr",
+    "fre": "fr",
+    "fr": "fr",
+    "deu": "de",
+    "ger": "de",
+    "de": "de",
+    "spa": "es",
+    "es": "es",
+    "jpn": "ja",
+    "jp": "ja",
+    "ja": "ja",
+    "kor": "ko",
+    "ko": "ko",
+    "tha": "th",
+    "th": "th",
+    "lao": "lo",
+    "lo": "lo",
+    "khm": "km",
+    "km": "km",
+    "mya": "my",
+    "bur": "my",
+    "my": "my",
+    "bod": "bo",
+    "tib": "bo",
+    "bo": "bo",
+    "dzo": "dz",
+    "dz": "dz",
+}
+
+_CER_LANGUAGES = {
+    "zh",
+    "ja",
+    "ko",
+    "th",
+    "lo",
+    "km",
+    "my",
+    "bo",
+    "dz",
+}
+
+
+def normalize_language_code(language: Optional[str]) -> str:
+    text = str(language or "").strip().lower()
+    if not text:
+        return ""
+    if text.startswith("<|") and text.endswith("|>"):
+        text = text[2:-2]
+    text = text.replace("-", "_")
+    base = text.split("_", 1)[0]
+    return _LANGUAGE_ALIASES.get(base, base)
+
+
+def speech_consistency_unit(language: Optional[str]) -> str:
+    normalized = normalize_language_code(language)
+    return "cer" if normalized in _CER_LANGUAGES else "wer"
+
+
+def whisper_language_code(language: Optional[str]) -> Optional[str]:
+    normalized = normalize_language_code(language)
+    return normalized or None
+
+
+def tokenization_metadata(language: Optional[str]) -> dict:
+    normalized = normalize_language_code(language)
+    warnings = []
+    if not normalized:
+        warnings.append("empty_language_code_defaulted_to_wer")
+    return {
+        "language_policy_version": LANGUAGE_POLICY_VERSION,
+        "target_language": str(language or ""),
+        "normalized_language": normalized,
+        "speech_consistency_unit": speech_consistency_unit(language),
+        "whisper_language": whisper_language_code(language),
+        "warnings": warnings,
+    }

openstbench-1.2.2/src/openstbench/latency/__init__.py

@@ -0,0 +1,39 @@
+# This file contains code adapted from SimulEval:
+# https://github.com/facebookresearch/SimulEval
+#
+# SimulEval is licensed under the Creative Commons Attribution-ShareAlike
+# 4.0 International License (CC BY-SA 4.0):
+# https://creativecommons.org/licenses/by-sa/4.0/
+#
+# The adapted portions in this file are distributed under CC BY-SA 4.0.
+# Modifications were made for OpenSTBench.
+
+from .agent import GenericAgent, AgentPipeline
+from .basics import ReadAction, WriteAction, TextSegment, SpeechSegment
+
+__all__ = [
+    "GenericAgent",
+    "AgentPipeline",
+    "ReadAction",
+    "WriteAction",
+    "TextSegment",
+    "SpeechSegment",
+    "LatencyEvaluator",
+    "register",
+]
+
+
+def __getattr__(name):
+    if name == "LatencyEvaluator":
+        from .cli import LatencyEvaluator
+
+        return LatencyEvaluator
+    if name == "register":
+        from .metrics import register
+
+        return register
+    raise AttributeError(f"module '{__name__}' has no attribute '{name}'")
+
+
+def __dir__():
+    return sorted(set(globals()) | set(__all__))

{openstbench-1.1.0 → openstbench-1.2.2}/src/openstbench/speech_quality_evaluator.py

@@ -1,202 +1,206 @@
-import os
-import torch
-import torchaudio
-import jiwer
-from pathlib import Path
-from typing import List, Dict, Optional, Union
-from tqdm import tqdm
-
-# Reuse existing loading utilities
+import os
+import torch
+import torchaudio
+import jiwer
+from pathlib import Path
+from typing import List, Dict, Optional, Union
+from tqdm import tqdm
+
+# Reuse existing loading utilities
 from ._model_loading import resolve_pretrained_source
+from .language_policy import normalize_language_code, speech_consistency_unit, whisper_language_code
 from .translation_evaluator import load_text_from_file_or_list, load_audio_from_folder
-
+
 try:
     import whisper
 except ImportError:
     whisper = None
 
-class SpeechQualityEvaluator:
-    DEFAULT_WHISPER_MODEL = "medium"
-    """
-    Speech Quality and Consistency Evaluator.
-    Specifically designed for:
-    1. UTMOS (Audio naturalness / perceived quality).
-    2. WER/CER Consistency Calculation (compares the model's generated text with its generated audio).
-    """
-    def __init__(self, 
-                 use_wer: bool = True,
-                 use_utmos: bool = True,
-                 whisper_model: str = DEFAULT_WHISPER_MODEL,
-                 whisper_language: Optional[str] = None,
-                 utmos_model_path: Optional[str] = None,
-                 utmos_ckpt_path: Optional[str] = None,
-                 device: Optional[str] = None):
-        
-        self.device = device if device else ("cuda" if torch.cuda.is_available() else "cpu")
-        self.use_wer = use_wer
-        self.use_utmos = use_utmos
-        
+try:
+    from opencc import OpenCC
+except ImportError:
+    OpenCC = None
+
+class SpeechQualityEvaluator:
+    DEFAULT_WHISPER_MODEL = "medium"
+    """
+    Speech Quality and Consistency Evaluator.
+    Specifically designed for:
+    1. UTMOS (Audio naturalness / perceived quality).
+    2. WER/CER Consistency Calculation (compares the model's generated text with its generated audio).
+    """
+    def __init__(self, 
+                 use_wer: bool = True,
+                 use_utmos: bool = True,
+                 whisper_model: str = DEFAULT_WHISPER_MODEL,
+                 whisper_language: Optional[str] = None,
+                 utmos_model_path: Optional[str] = None,
+                 utmos_ckpt_path: Optional[str] = None,
+                 device: Optional[str] = None):
+        
+        self.device = device if device else ("cuda" if torch.cuda.is_available() else "cpu")
+        self.use_wer = use_wer
+        self.use_utmos = use_utmos
+        
         self.whisper_model_name = whisper_model
         self.whisper_language = self._normalize_whisper_language(whisper_language)
         self.utmos_path = utmos_model_path
         self.utmos_ckpt = utmos_ckpt_path
-        
-        self.whisper_model = None
-        self.utmos_model = None
-        
-        if self.use_wer:
-            self.wer_transform = jiwer.Compose([
-                jiwer.ToLowerCase(),
-                jiwer.RemovePunctuation(),
-                jiwer.RemoveMultipleSpaces(),
-                jiwer.Strip(),
-                jiwer.RemoveEmptyStrings(),
-            ])
-
+        self.zh_converter = OpenCC("t2s") if OpenCC is not None else None
+        self._warned_missing_opencc = False
+        
+        self.whisper_model = None
+        self.utmos_model = None
+        
+        if self.use_wer:
+            self.wer_transform = jiwer.Compose([
+                jiwer.ToLowerCase(),
+                jiwer.RemovePunctuation(),
+                jiwer.RemoveMultipleSpaces(),
+                jiwer.Strip(),
+                jiwer.RemoveEmptyStrings(),
+            ])
+
     @staticmethod
     def _normalize_whisper_language(language: Optional[str]) -> Optional[str]:
-        if language is None:
-            return None
-        normalized = str(language).strip().lower()
-        if normalized in {"zh", "zh-cn", "cmn", "<|cmn|>"}:
-            return "zh"
-        if normalized in {"en", "eng", "<|eng|>"}:
-            return "en"
-        if normalized in {"ja", "jp", "jpn"}:
-            return "ja"
-        if normalized in {"ko", "kor"}:
-            return "ko"
-        return normalized or None
-
-    def _load_whisper(self):
-        if self.whisper_model is None and self.use_wer:
-            if not whisper:
-                print("⚠️ Whisper is not installed, skipping load.")
-                self.use_wer = False
-                return
-            print(f"⏳ Loading Whisper ({self.whisper_model_name})...")
-            model_source, source_kind = resolve_pretrained_source(
-                self.whisper_model_name,
-                fallback_source=self.DEFAULT_WHISPER_MODEL,
-            )
-            print(f"Loading Whisper ({source_kind}) from {model_source}...")
-            self.whisper_model = whisper.load_model(model_source, device=self.device)
-
-    def _load_utmos(self):
-        if self.utmos_model is None and self.use_utmos:
-            print("⏳ Loading UTMOS model...")
-            try:
-                source = "github"
-                repo_or_dir = "tarepan/SpeechMOS"
-                if self.utmos_path and os.path.exists(self.utmos_path):
-                    source = "local"
-                    repo_or_dir = self.utmos_path
-
-                load_pretrained = (self.utmos_ckpt is None)
-                self.utmos_model = torch.hub.load(
-                    repo_or_dir, "utmos22_strong", source=source, trust_repo=True, pretrained=load_pretrained
-                )
-                
-                if self.utmos_ckpt and os.path.exists(self.utmos_ckpt):
-                    state_dict = torch.load(self.utmos_ckpt, map_location=self.device)
-                    self.utmos_model.load_state_dict(state_dict)
-                    
-                self.utmos_model.to(self.device).eval()
-            except Exception as e:
-                print(f"⚠️ UTMOS model failed to load: {e}")
-                self.use_utmos = False
-
-    def _transcribe(self, audio_paths: List[str]) -> List[str]:
-        self._load_whisper()
-        if not self.whisper_model:
-            return [""] * len(audio_paths)
-
-        transcribe_kwargs = {
-            "fp16": torch.cuda.is_available() and "cuda" in self.device,
-            "task": "transcribe",
-        }
-        if self.whisper_language:
-            transcribe_kwargs["language"] = self.whisper_language
-
-        results = []
-        for path in tqdm(audio_paths, desc="Whisper transcription"):
-            res = self.whisper_model.transcribe(path, **transcribe_kwargs)
-            results.append(res["text"].strip())
-        return results
-
-    def _compute_utmos(self, audio_paths: List[str]) -> float:
-        self._load_utmos()
-        if not self.utmos_model: return 0.0
-        
-        scores = []
-        target_sr = 16000
-        for path in tqdm(audio_paths, desc="🎧 Computing UTMOS"):
-            try:
-                wave, sr = torchaudio.load(path)
-                if sr != target_sr:
-                    wave = torchaudio.functional.resample(wave, sr, target_sr)
-                if wave.shape[0] > 1:
-                    wave = torch.mean(wave, dim=0, keepdim=True)
-                
-                wave = wave.to(self.device)
-                with torch.no_grad():
-                    s = self.utmos_model(wave, target_sr)
-                    scores.append(s.item())
-            except Exception as e:
-                print(f"⚠️ UTMOS Error on {path}: {e}")
-                
-        return sum(scores) / len(scores) if scores else 0.0
-
+        return whisper_language_code(language)
+
+    def _load_whisper(self):
+        if self.whisper_model is None and self.use_wer:
+            if not whisper:
+                print("⚠️ Whisper is not installed, skipping load.")
+                self.use_wer = False
+                return
+            print(f"⏳ Loading Whisper ({self.whisper_model_name})...")
+            model_source, source_kind = resolve_pretrained_source(
+                self.whisper_model_name,
+                fallback_source=self.DEFAULT_WHISPER_MODEL,
+            )
+            print(f"Loading Whisper ({source_kind}) from {model_source}...")
+            self.whisper_model = whisper.load_model(model_source, device=self.device)
+
+    def _load_utmos(self):
+        if self.utmos_model is None and self.use_utmos:
+            print("⏳ Loading UTMOS model...")
+            try:
+                source = "github"
+                repo_or_dir = "tarepan/SpeechMOS"
+                if self.utmos_path and os.path.exists(self.utmos_path):
+                    source = "local"
+                    repo_or_dir = self.utmos_path
+
+                load_pretrained = (self.utmos_ckpt is None)
+                self.utmos_model = torch.hub.load(
+                    repo_or_dir, "utmos22_strong", source=source, trust_repo=True, pretrained=load_pretrained
+                )
+                
+                if self.utmos_ckpt and os.path.exists(self.utmos_ckpt):
+                    state_dict = torch.load(self.utmos_ckpt, map_location=self.device)
+                    self.utmos_model.load_state_dict(state_dict)
+                    
+                self.utmos_model.to(self.device).eval()
+            except Exception as e:
+                print(f"⚠️ UTMOS model failed to load: {e}")
+                self.use_utmos = False
+
+    def _transcribe(self, audio_paths: List[str]) -> List[str]:
+        self._load_whisper()
+        if not self.whisper_model:
+            return [""] * len(audio_paths)
+
+        transcribe_kwargs = {
+            "fp16": torch.cuda.is_available() and "cuda" in self.device,
+            "task": "transcribe",
+        }
+        if self.whisper_language:
+            transcribe_kwargs["language"] = self.whisper_language
+
+        results = []
+        for path in tqdm(audio_paths, desc="Whisper transcription"):
+            res = self.whisper_model.transcribe(path, **transcribe_kwargs)
+            results.append(res["text"].strip())
+        return results
+
+    def _compute_utmos(self, audio_paths: List[str]) -> float:
+        self._load_utmos()
+        if not self.utmos_model: return 0.0
+        
+        scores = []
+        target_sr = 16000
+        for path in tqdm(audio_paths, desc="🎧 Computing UTMOS"):
+            try:
+                wave, sr = torchaudio.load(path)
+                if sr != target_sr:
+                    wave = torchaudio.functional.resample(wave, sr, target_sr)
+                if wave.shape[0] > 1:
+                    wave = torch.mean(wave, dim=0, keepdim=True)
+                
+                wave = wave.to(self.device)
+                with torch.no_grad():
+                    s = self.utmos_model(wave, target_sr)
+                    scores.append(s.item())
+            except Exception as e:
+                print(f"⚠️ UTMOS Error on {path}: {e}")
+                
+        return sum(scores) / len(scores) if scores else 0.0
+
     def _preprocess_for_wer(self, text: str, lang: str) -> str:
+        normalized_lang = normalize_language_code(lang)
         text = self.wer_transform(text)
-        if lang in ["zh", "ja", "ko"]:
+        if normalized_lang == "zh":
+            if self.zh_converter is not None:
+                text = self.zh_converter.convert(text)
+            elif not self._warned_missing_opencc:
+                print("Warning: OpenCC is not installed; Chinese CER will not normalize traditional/simplified variants.")
+                self._warned_missing_opencc = True
+        if speech_consistency_unit(normalized_lang) == "cer":
             text = text.replace(" ", "")
             return " ".join(list(text))
         return text
-
-    def evaluate_all(self, 
-                     target_audio: Union[List[str], str],
-                     target_text: Optional[Union[List[str], str]] = None,
-                     target_lang: str = "en") -> Dict[str, float]:
-        """
-        :param target_audio: List of paths or directory to the model-generated audio.
-        :param target_text: Text generated synchronously by the model, used as reference for WER/CER calculation.
-        :param target_lang: Target language for both audio and text.
-        """
-        results = {}
-        print(f"\n--- Starting Speech Quality Evaluation (Target Lang: {target_lang}) ---")
-        
-        # Load target audio
-        if isinstance(target_audio, str) and os.path.exists(target_audio) and os.path.isdir(target_audio):
-            audio_paths = load_audio_from_folder(target_audio)
-        else:
-            audio_paths = target_audio if isinstance(target_audio, list) else [target_audio]
-
-        # 1. evaluate UTMOS
-        if self.use_utmos:
-            print("   ➤ Computing UTMOS (Audio Naturalness)...")
-            results["UTMOS"] = round(self._compute_utmos(audio_paths), 4)
-
-        # 2. evaluate Consistency WER/CER
-        if self.use_wer:
-            if not target_text:
-                print(" ⚠️ target_text (synchronously generated text) not provided. Text-speech consistency cannot be evaluated, skipping WER/CER.")
-            else:
-                texts = load_text_from_file_or_list(target_text, "target_text")
-                if len(texts) != len(audio_paths):
-                    raise ValueError(f"Number of generated text lines ({len(texts)}) does not match number of audio files ({len(audio_paths)})!")
-                
-                print("   ➤ Transcribing audio, preparing consistency calculation...")
-                asr_texts = self._transcribe(audio_paths)
-                
-                clean_refs = [self._preprocess_for_wer(t, target_lang) for t in texts]
-                clean_hyps = [self._preprocess_for_wer(t, target_lang) for t in asr_texts]
-                
+
+    def evaluate_all(self, 
+                     target_audio: Union[List[str], str],
+                     target_text: Optional[Union[List[str], str]] = None,
+                     target_lang: str = "en") -> Dict[str, float]:
+        """
+        :param target_audio: List of paths or directory to the model-generated audio.
+        :param target_text: Text generated synchronously by the model, used as reference for WER/CER calculation.
+        :param target_lang: Target language for both audio and text.
+        """
+        results = {}
+        print(f"\n--- Starting Speech Quality Evaluation (Target Lang: {target_lang}) ---")
+        
+        # Load target audio
+        if isinstance(target_audio, str) and os.path.exists(target_audio) and os.path.isdir(target_audio):
+            audio_paths = load_audio_from_folder(target_audio)
+        else:
+            audio_paths = target_audio if isinstance(target_audio, list) else [target_audio]
+
+        # 1. evaluate UTMOS
+        if self.use_utmos:
+            print("   ➤ Computing UTMOS (Audio Naturalness)...")
+            results["UTMOS"] = round(self._compute_utmos(audio_paths), 4)
+
+        # 2. evaluate Consistency WER/CER
+        if self.use_wer:
+            if not target_text:
+                print(" ⚠️ target_text (synchronously generated text) not provided. Text-speech consistency cannot be evaluated, skipping WER/CER.")
+            else:
+                texts = load_text_from_file_or_list(target_text, "target_text")
+                if len(texts) != len(audio_paths):
+                    raise ValueError(f"Number of generated text lines ({len(texts)}) does not match number of audio files ({len(audio_paths)})!")
+                
+                print("   ➤ Transcribing audio, preparing consistency calculation...")
+                asr_texts = self._transcribe(audio_paths)
+                
+                clean_refs = [self._preprocess_for_wer(t, target_lang) for t in texts]
+                clean_hyps = [self._preprocess_for_wer(t, target_lang) for t in asr_texts]
+                
                 if clean_refs:
                     error_rate = jiwer.wer(clean_refs, clean_hyps)
-                    metric_name = "CER_Consistency" if target_lang in ['zh', 'ja', 'ko'] else "WER_Consistency"
+                    metric_name = "CER_Consistency" if speech_consistency_unit(target_lang) == "cer" else "WER_Consistency"
                     results[metric_name] = round(error_rate, 4)
-                else:
-                    print("   ⚠️ The provided generated text is empty after cleaning, cannot compute distribution error rate.") 
-
-        return results
+                else:
+                    print("   ⚠️ The provided generated text is empty after cleaning, cannot compute distribution error rate.") 
+
+        return results

{openstbench-1.1.0 → openstbench-1.2.2}/src/openstbench/translation_evaluator.py

@@ -84,8 +84,9 @@ def load_audio_from_folder(folder_path: str, extensions: tuple = (".wav", ".mp3"
 
 # ==================== Evaluator Core Class ====================
 
-DEFAULT_COMET_MODEL = "Unbabel/wmt22-comet-da"
+DEFAULT_COMET_MODEL = "Unbabel/wmt22-cometkiwi-da"
 DEFAULT_BLEURT_MODEL = "lucadiliello/BLEURT-20"
+COMET_QE_METRIC_NAME = "COMETKiwi"
 
 class TranslationEvaluator:
     """
@@ -237,7 +238,7 @@ class TranslationEvaluator:
         Args:
             reference: Ground truth benchmark translation reference from the dataset (List or file path).
             target_text: Direct text translation output from the translation model (List or file path).
-            source: Optional original source text (mandatory for COMET).
+            source: Optional original source text (mandatory for COMETKiwi/QE).
             target_lang: Target language, affects BLEU Tokenizer selection.
         """
         results = {}
@@ -276,13 +277,19 @@ class TranslationEvaluator:
             except Exception as e:
                 results["BLEURT"] = -1.0
 
-        # 4. COMET
-        if self.use_comet and self.comet and final_src:
+        # 4. COMETKiwi reference-free QE. References may still be passed for
+        # BLEU/chrF++/BLEURT, but COMETKiwi must only receive source and MT.
+        if self.use_comet and self.comet:
+            if not final_src:
+                results[COMET_QE_METRIC_NAME] = -1.0
+                return {k: round(v, 4) if v >= 0 else v for k, v in results.items()}
+            if len(final_src) != len(final_text):
+                raise ValueError("Source and Target Text have different lengths")
             try:
-                data = [{"src": s, "mt": t, "ref": r} for s, t, r in zip(final_src, final_text, final_ref)]
+                data = [{"src": s, "mt": t} for s, t in zip(final_src, final_text)]
                 gpus = 1 if self.device.startswith("cuda") else 0
-                results["COMET"] = self.comet.predict(data, batch_size=8, gpus=gpus).system_score
+                results[COMET_QE_METRIC_NAME] = self.comet.predict(data, batch_size=8, gpus=gpus).system_score
             except Exception as e:
-                results["COMET"] = -1.0
+                results[COMET_QE_METRIC_NAME] = -1.0
         
         return {k: round(v, 4) if v >= 0 else v for k, v in results.items()}

openstbench-1.1.0/src/openstbench/latency/__init__.py

@@ -1,14 +0,0 @@
-# This file contains code adapted from SimulEval:
-# https://github.com/facebookresearch/SimulEval
-#
-# SimulEval is licensed under the Creative Commons Attribution-ShareAlike
-# 4.0 International License (CC BY-SA 4.0):
-# https://creativecommons.org/licenses/by-sa/4.0/
-#
-# The adapted portions in this file are distributed under CC BY-SA 4.0.
-# Modifications were made for OpenSTBench.
-
-from .agent import GenericAgent, AgentPipeline
-from .basics import ReadAction, WriteAction, TextSegment, SpeechSegment
-from .cli import LatencyEvaluator
-from .metrics import register # Allows users to register custom Scorers