OpenSTBench 1.2.0__tar.gz → 1.2.3__tar.gz

{openstbench-1.2.0/src/OpenSTBench.egg-info → openstbench-1.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: OpenSTBench
-Version: 1.2.0
+Version: 1.2.3
 Summary: Unified multidimensional evaluation toolkit for S2TT and S2ST systems in offline and streaming speech translation settings
 Author-email: Yanjie An <691476922@qq.com>
 License: MIT

{openstbench-1.2.0 → openstbench-1.2.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "OpenSTBench"
-version = "1.2.0"  
+version = "1.2.3"  
 description = "Unified multidimensional evaluation toolkit for S2TT and S2ST systems in offline and streaming speech translation settings"
 readme = {file = "README.md", content-type = "text/markdown"}
 requires-python = ">=3.9,<3.11"

{openstbench-1.2.0 → openstbench-1.2.3/src/OpenSTBench.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: OpenSTBench
-Version: 1.2.0
+Version: 1.2.3
 Summary: Unified multidimensional evaluation toolkit for S2TT and S2ST systems in offline and streaming speech translation settings
 Author-email: Yanjie An <691476922@qq.com>
 License: MIT

{openstbench-1.2.0 → openstbench-1.2.3}/src/OpenSTBench.egg-info/SOURCES.txt

@@ -9,6 +9,7 @@ src/OpenSTBench.egg-info/top_level.txt
 src/openstbench/__init__.py
 src/openstbench/_model_loading.py
 src/openstbench/emotion_evaluator.py
+src/openstbench/language_policy.py
 src/openstbench/paralinguistic_evaluator.py
 src/openstbench/speaker_similarity_evaluator.py
 src/openstbench/speech_quality_evaluator.py

{openstbench-1.2.0 → openstbench-1.2.3}/src/openstbench/__init__.py

@@ -9,7 +9,7 @@ actually accessed.
 from importlib import import_module
 from typing import Dict, Tuple
 
-__version__ = "1.2.0"
+__version__ = "1.2.3"
 
 __all__ = [
     "TranslationEvaluator",
@@ -35,6 +35,8 @@ __all__ = [
     "AgentPipeline",
     "ReadAction",
     "WriteAction",
+    "TextSegment",
+    "SpeechSegment",
     "load_text_from_file_or_list",
     "load_audio_from_folder",
     # "Dataset",
@@ -87,7 +89,7 @@ _EXPORT_SPECS: Tuple[Tuple[str, Tuple[str, ...]], ...] = (
     #     ),
     # ),
     ("latency.agent", ("AgentPipeline", "GenericAgent")),
-    ("latency.basics", ("ReadAction", "WriteAction")),
+    ("latency.basics", ("ReadAction", "WriteAction", "TextSegment", "SpeechSegment")),
     ("latency.cli", ("LatencyEvaluator",)),
 )
 

openstbench-1.2.3/src/openstbench/language_policy.py

@@ -0,0 +1,91 @@
+from __future__ import annotations
+
+from typing import Optional
+
+
+LANGUAGE_POLICY_VERSION = "2026-07-fleurs-wer-cer-v1"
+
+_LANGUAGE_ALIASES = {
+    "cmn": "zh",
+    "zho": "zh",
+    "chi": "zh",
+    "zh": "zh",
+    "yue": "zh",
+    "eng": "en",
+    "en": "en",
+    "fra": "fr",
+    "fre": "fr",
+    "fr": "fr",
+    "deu": "de",
+    "ger": "de",
+    "de": "de",
+    "spa": "es",
+    "es": "es",
+    "jpn": "ja",
+    "jp": "ja",
+    "ja": "ja",
+    "kor": "ko",
+    "ko": "ko",
+    "tha": "th",
+    "th": "th",
+    "lao": "lo",
+    "lo": "lo",
+    "khm": "km",
+    "km": "km",
+    "mya": "my",
+    "bur": "my",
+    "my": "my",
+    "bod": "bo",
+    "tib": "bo",
+    "bo": "bo",
+    "dzo": "dz",
+    "dz": "dz",
+}
+
+_CER_LANGUAGES = {
+    "zh",
+    "ja",
+    "ko",
+    "th",
+    "lo",
+    "km",
+    "my",
+    "bo",
+    "dz",
+}
+
+
+def normalize_language_code(language: Optional[str]) -> str:
+    text = str(language or "").strip().lower()
+    if not text:
+        return ""
+    if text.startswith("<|") and text.endswith("|>"):
+        text = text[2:-2]
+    text = text.replace("-", "_")
+    base = text.split("_", 1)[0]
+    return _LANGUAGE_ALIASES.get(base, base)
+
+
+def speech_consistency_unit(language: Optional[str]) -> str:
+    normalized = normalize_language_code(language)
+    return "cer" if normalized in _CER_LANGUAGES else "wer"
+
+
+def whisper_language_code(language: Optional[str]) -> Optional[str]:
+    normalized = normalize_language_code(language)
+    return normalized or None
+
+
+def tokenization_metadata(language: Optional[str]) -> dict:
+    normalized = normalize_language_code(language)
+    warnings = []
+    if not normalized:
+        warnings.append("empty_language_code_defaulted_to_wer")
+    return {
+        "language_policy_version": LANGUAGE_POLICY_VERSION,
+        "target_language": str(language or ""),
+        "normalized_language": normalized,
+        "speech_consistency_unit": speech_consistency_unit(language),
+        "whisper_language": whisper_language_code(language),
+        "warnings": warnings,
+    }

openstbench-1.2.3/src/openstbench/latency/__init__.py

@@ -0,0 +1,39 @@
+# This file contains code adapted from SimulEval:
+# https://github.com/facebookresearch/SimulEval
+#
+# SimulEval is licensed under the Creative Commons Attribution-ShareAlike
+# 4.0 International License (CC BY-SA 4.0):
+# https://creativecommons.org/licenses/by-sa/4.0/
+#
+# The adapted portions in this file are distributed under CC BY-SA 4.0.
+# Modifications were made for OpenSTBench.
+
+from .agent import GenericAgent, AgentPipeline
+from .basics import ReadAction, WriteAction, TextSegment, SpeechSegment
+
+__all__ = [
+    "GenericAgent",
+    "AgentPipeline",
+    "ReadAction",
+    "WriteAction",
+    "TextSegment",
+    "SpeechSegment",
+    "LatencyEvaluator",
+    "register",
+]
+
+
+def __getattr__(name):
+    if name == "LatencyEvaluator":
+        from .cli import LatencyEvaluator
+
+        return LatencyEvaluator
+    if name == "register":
+        from .metrics import register
+
+        return register
+    raise AttributeError(f"module '{__name__}' has no attribute '{name}'")
+
+
+def __dir__():
+    return sorted(set(globals()) | set(__all__))

{openstbench-1.2.0 → openstbench-1.2.3}/src/openstbench/speech_quality_evaluator.py

@@ -1,15 +1,16 @@
-import os
-import torch
-import torchaudio
-import jiwer
-from pathlib import Path
-from typing import List, Dict, Optional, Union
-from tqdm import tqdm
-
-# Reuse existing loading utilities
+import os
+import torch
+import torchaudio
+import jiwer
+from pathlib import Path
+from typing import List, Dict, Optional, Union
+from tqdm import tqdm
+
+# Reuse existing loading utilities
 from ._model_loading import resolve_pretrained_source
+from .language_policy import normalize_language_code, speech_consistency_unit, whisper_language_code
 from .translation_evaluator import load_text_from_file_or_list, load_audio_from_folder
-
+
 try:
     import whisper
 except ImportError:
@@ -19,143 +20,132 @@ try:
     from opencc import OpenCC
 except ImportError:
     OpenCC = None
-
-class SpeechQualityEvaluator:
-    DEFAULT_WHISPER_MODEL = "medium"
-    """
-    Speech Quality and Consistency Evaluator.
-    Specifically designed for:
-    1. UTMOS (Audio naturalness / perceived quality).
-    2. WER/CER Consistency Calculation (compares the model's generated text with its generated audio).
-    """
-    def __init__(self, 
-                 use_wer: bool = True,
-                 use_utmos: bool = True,
-                 whisper_model: str = DEFAULT_WHISPER_MODEL,
-                 whisper_language: Optional[str] = None,
-                 utmos_model_path: Optional[str] = None,
-                 utmos_ckpt_path: Optional[str] = None,
-                 device: Optional[str] = None):
-        
-        self.device = device if device else ("cuda" if torch.cuda.is_available() else "cpu")
-        self.use_wer = use_wer
-        self.use_utmos = use_utmos
-        
+
+class SpeechQualityEvaluator:
+    DEFAULT_WHISPER_MODEL = "medium"
+    """
+    Speech Quality and Consistency Evaluator.
+    Specifically designed for:
+    1. UTMOS (Audio naturalness / perceived quality).
+    2. WER/CER Consistency Calculation (compares the model's generated text with its generated audio).
+    """
+    def __init__(self, 
+                 use_wer: bool = True,
+                 use_utmos: bool = True,
+                 whisper_model: str = DEFAULT_WHISPER_MODEL,
+                 whisper_language: Optional[str] = None,
+                 utmos_model_path: Optional[str] = None,
+                 utmos_ckpt_path: Optional[str] = None,
+                 device: Optional[str] = None):
+        
+        self.device = device if device else ("cuda" if torch.cuda.is_available() else "cpu")
+        self.use_wer = use_wer
+        self.use_utmos = use_utmos
+        
         self.whisper_model_name = whisper_model
         self.whisper_language = self._normalize_whisper_language(whisper_language)
         self.utmos_path = utmos_model_path
         self.utmos_ckpt = utmos_ckpt_path
         self.zh_converter = OpenCC("t2s") if OpenCC is not None else None
         self._warned_missing_opencc = False
-        
-        self.whisper_model = None
-        self.utmos_model = None
-        
-        if self.use_wer:
-            self.wer_transform = jiwer.Compose([
-                jiwer.ToLowerCase(),
-                jiwer.RemovePunctuation(),
-                jiwer.RemoveMultipleSpaces(),
-                jiwer.Strip(),
-                jiwer.RemoveEmptyStrings(),
-            ])
-
+        
+        self.whisper_model = None
+        self.utmos_model = None
+        
+        if self.use_wer:
+            self.wer_transform = jiwer.Compose([
+                jiwer.ToLowerCase(),
+                jiwer.RemovePunctuation(),
+                jiwer.RemoveMultipleSpaces(),
+                jiwer.Strip(),
+                jiwer.RemoveEmptyStrings(),
+            ])
+
     @staticmethod
     def _normalize_whisper_language(language: Optional[str]) -> Optional[str]:
-        if language is None:
-            return None
-        normalized = str(language).strip().lower()
-        if normalized in {"zh", "zh-cn", "cmn", "<|cmn|>"}:
-            return "zh"
-        if normalized in {"en", "eng", "<|eng|>"}:
-            return "en"
-        if normalized in {"ja", "jp", "jpn"}:
-            return "ja"
-        if normalized in {"ko", "kor"}:
-            return "ko"
-        return normalized or None
-
-    def _load_whisper(self):
-        if self.whisper_model is None and self.use_wer:
-            if not whisper:
-                print("⚠️ Whisper is not installed, skipping load.")
-                self.use_wer = False
-                return
-            print(f"⏳ Loading Whisper ({self.whisper_model_name})...")
-            model_source, source_kind = resolve_pretrained_source(
-                self.whisper_model_name,
-                fallback_source=self.DEFAULT_WHISPER_MODEL,
-            )
-            print(f"Loading Whisper ({source_kind}) from {model_source}...")
-            self.whisper_model = whisper.load_model(model_source, device=self.device)
-
-    def _load_utmos(self):
-        if self.utmos_model is None and self.use_utmos:
-            print("⏳ Loading UTMOS model...")
-            try:
-                source = "github"
-                repo_or_dir = "tarepan/SpeechMOS"
-                if self.utmos_path and os.path.exists(self.utmos_path):
-                    source = "local"
-                    repo_or_dir = self.utmos_path
-
-                load_pretrained = (self.utmos_ckpt is None)
-                self.utmos_model = torch.hub.load(
-                    repo_or_dir, "utmos22_strong", source=source, trust_repo=True, pretrained=load_pretrained
-                )
-                
-                if self.utmos_ckpt and os.path.exists(self.utmos_ckpt):
-                    state_dict = torch.load(self.utmos_ckpt, map_location=self.device)
-                    self.utmos_model.load_state_dict(state_dict)
-                    
-                self.utmos_model.to(self.device).eval()
-            except Exception as e:
-                print(f"⚠️ UTMOS model failed to load: {e}")
-                self.use_utmos = False
-
-    def _transcribe(self, audio_paths: List[str]) -> List[str]:
-        self._load_whisper()
-        if not self.whisper_model:
-            return [""] * len(audio_paths)
-
-        transcribe_kwargs = {
-            "fp16": torch.cuda.is_available() and "cuda" in self.device,
-            "task": "transcribe",
-        }
-        if self.whisper_language:
-            transcribe_kwargs["language"] = self.whisper_language
-
-        results = []
-        for path in tqdm(audio_paths, desc="Whisper transcription"):
-            res = self.whisper_model.transcribe(path, **transcribe_kwargs)
-            results.append(res["text"].strip())
-        return results
-
-    def _compute_utmos(self, audio_paths: List[str]) -> float:
-        self._load_utmos()
-        if not self.utmos_model: return 0.0
-        
-        scores = []
-        target_sr = 16000
-        for path in tqdm(audio_paths, desc="🎧 Computing UTMOS"):
-            try:
-                wave, sr = torchaudio.load(path)
-                if sr != target_sr:
-                    wave = torchaudio.functional.resample(wave, sr, target_sr)
-                if wave.shape[0] > 1:
-                    wave = torch.mean(wave, dim=0, keepdim=True)
-                
-                wave = wave.to(self.device)
-                with torch.no_grad():
-                    s = self.utmos_model(wave, target_sr)
-                    scores.append(s.item())
-            except Exception as e:
-                print(f"⚠️ UTMOS Error on {path}: {e}")
-                
-        return sum(scores) / len(scores) if scores else 0.0
-
+        return whisper_language_code(language)
+
+    def _load_whisper(self):
+        if self.whisper_model is None and self.use_wer:
+            if not whisper:
+                print("⚠️ Whisper is not installed, skipping load.")
+                self.use_wer = False
+                return
+            print(f"⏳ Loading Whisper ({self.whisper_model_name})...")
+            model_source, source_kind = resolve_pretrained_source(
+                self.whisper_model_name,
+                fallback_source=self.DEFAULT_WHISPER_MODEL,
+            )
+            print(f"Loading Whisper ({source_kind}) from {model_source}...")
+            self.whisper_model = whisper.load_model(model_source, device=self.device)
+
+    def _load_utmos(self):
+        if self.utmos_model is None and self.use_utmos:
+            print("⏳ Loading UTMOS model...")
+            try:
+                source = "github"
+                repo_or_dir = "tarepan/SpeechMOS"
+                if self.utmos_path and os.path.exists(self.utmos_path):
+                    source = "local"
+                    repo_or_dir = self.utmos_path
+
+                load_pretrained = (self.utmos_ckpt is None)
+                self.utmos_model = torch.hub.load(
+                    repo_or_dir, "utmos22_strong", source=source, trust_repo=True, pretrained=load_pretrained
+                )
+                
+                if self.utmos_ckpt and os.path.exists(self.utmos_ckpt):
+                    state_dict = torch.load(self.utmos_ckpt, map_location=self.device)
+                    self.utmos_model.load_state_dict(state_dict)
+                    
+                self.utmos_model.to(self.device).eval()
+            except Exception as e:
+                print(f"⚠️ UTMOS model failed to load: {e}")
+                self.use_utmos = False
+
+    def _transcribe(self, audio_paths: List[str]) -> List[str]:
+        self._load_whisper()
+        if not self.whisper_model:
+            return [""] * len(audio_paths)
+
+        transcribe_kwargs = {
+            "fp16": torch.cuda.is_available() and "cuda" in self.device,
+            "task": "transcribe",
+        }
+        if self.whisper_language:
+            transcribe_kwargs["language"] = self.whisper_language
+
+        results = []
+        for path in tqdm(audio_paths, desc="Whisper transcription"):
+            res = self.whisper_model.transcribe(path, **transcribe_kwargs)
+            results.append(res["text"].strip())
+        return results
+
+    def _compute_utmos(self, audio_paths: List[str]) -> float:
+        self._load_utmos()
+        if not self.utmos_model: return 0.0
+        
+        scores = []
+        target_sr = 16000
+        for path in tqdm(audio_paths, desc="🎧 Computing UTMOS"):
+            try:
+                wave, sr = torchaudio.load(path)
+                if sr != target_sr:
+                    wave = torchaudio.functional.resample(wave, sr, target_sr)
+                if wave.shape[0] > 1:
+                    wave = torch.mean(wave, dim=0, keepdim=True)
+                
+                wave = wave.to(self.device)
+                with torch.no_grad():
+                    s = self.utmos_model(wave, target_sr)
+                    scores.append(s.item())
+            except Exception as e:
+                print(f"⚠️ UTMOS Error on {path}: {e}")
+                
+        return sum(scores) / len(scores) if scores else 0.0
+
     def _preprocess_for_wer(self, text: str, lang: str) -> str:
-        normalized_lang = self._normalize_whisper_language(lang) or str(lang).strip().lower()
+        normalized_lang = normalize_language_code(lang)
         text = self.wer_transform(text)
         if normalized_lang == "zh":
             if self.zh_converter is not None:
@@ -163,55 +153,54 @@ class SpeechQualityEvaluator:
             elif not self._warned_missing_opencc:
                 print("Warning: OpenCC is not installed; Chinese CER will not normalize traditional/simplified variants.")
                 self._warned_missing_opencc = True
-        if normalized_lang in ["zh", "ja", "ko"]:
+        if speech_consistency_unit(normalized_lang) == "cer":
             text = text.replace(" ", "")
             return " ".join(list(text))
         return text
-
-    def evaluate_all(self, 
-                     target_audio: Union[List[str], str],
-                     target_text: Optional[Union[List[str], str]] = None,
-                     target_lang: str = "en") -> Dict[str, float]:
-        """
-        :param target_audio: List of paths or directory to the model-generated audio.
-        :param target_text: Text generated synchronously by the model, used as reference for WER/CER calculation.
-        :param target_lang: Target language for both audio and text.
-        """
-        results = {}
-        print(f"\n--- Starting Speech Quality Evaluation (Target Lang: {target_lang}) ---")
-        
-        # Load target audio
-        if isinstance(target_audio, str) and os.path.exists(target_audio) and os.path.isdir(target_audio):
-            audio_paths = load_audio_from_folder(target_audio)
-        else:
-            audio_paths = target_audio if isinstance(target_audio, list) else [target_audio]
-
-        # 1. evaluate UTMOS
-        if self.use_utmos:
-            print("   ➤ Computing UTMOS (Audio Naturalness)...")
-            results["UTMOS"] = round(self._compute_utmos(audio_paths), 4)
-
-        # 2. evaluate Consistency WER/CER
-        if self.use_wer:
-            if not target_text:
-                print(" ⚠️ target_text (synchronously generated text) not provided. Text-speech consistency cannot be evaluated, skipping WER/CER.")
-            else:
-                texts = load_text_from_file_or_list(target_text, "target_text")
-                if len(texts) != len(audio_paths):
-                    raise ValueError(f"Number of generated text lines ({len(texts)}) does not match number of audio files ({len(audio_paths)})!")
-                
-                print("   ➤ Transcribing audio, preparing consistency calculation...")
-                asr_texts = self._transcribe(audio_paths)
-                
-                clean_refs = [self._preprocess_for_wer(t, target_lang) for t in texts]
-                clean_hyps = [self._preprocess_for_wer(t, target_lang) for t in asr_texts]
-                
+
+    def evaluate_all(self, 
+                     target_audio: Union[List[str], str],
+                     target_text: Optional[Union[List[str], str]] = None,
+                     target_lang: str = "en") -> Dict[str, float]:
+        """
+        :param target_audio: List of paths or directory to the model-generated audio.
+        :param target_text: Text generated synchronously by the model, used as reference for WER/CER calculation.
+        :param target_lang: Target language for both audio and text.
+        """
+        results = {}
+        print(f"\n--- Starting Speech Quality Evaluation (Target Lang: {target_lang}) ---")
+        
+        # Load target audio
+        if isinstance(target_audio, str) and os.path.exists(target_audio) and os.path.isdir(target_audio):
+            audio_paths = load_audio_from_folder(target_audio)
+        else:
+            audio_paths = target_audio if isinstance(target_audio, list) else [target_audio]
+
+        # 1. evaluate UTMOS
+        if self.use_utmos:
+            print("   ➤ Computing UTMOS (Audio Naturalness)...")
+            results["UTMOS"] = round(self._compute_utmos(audio_paths), 4)
+
+        # 2. evaluate Consistency WER/CER
+        if self.use_wer:
+            if not target_text:
+                print(" ⚠️ target_text (synchronously generated text) not provided. Text-speech consistency cannot be evaluated, skipping WER/CER.")
+            else:
+                texts = load_text_from_file_or_list(target_text, "target_text")
+                if len(texts) != len(audio_paths):
+                    raise ValueError(f"Number of generated text lines ({len(texts)}) does not match number of audio files ({len(audio_paths)})!")
+                
+                print("   ➤ Transcribing audio, preparing consistency calculation...")
+                asr_texts = self._transcribe(audio_paths)
+                
+                clean_refs = [self._preprocess_for_wer(t, target_lang) for t in texts]
+                clean_hyps = [self._preprocess_for_wer(t, target_lang) for t in asr_texts]
+                
                 if clean_refs:
                     error_rate = jiwer.wer(clean_refs, clean_hyps)
-                    normalized_target_lang = self._normalize_whisper_language(target_lang) or str(target_lang).strip().lower()
-                    metric_name = "CER_Consistency" if normalized_target_lang in ['zh', 'ja', 'ko'] else "WER_Consistency"
+                    metric_name = "CER_Consistency" if speech_consistency_unit(target_lang) == "cer" else "WER_Consistency"
                     results[metric_name] = round(error_rate, 4)
-                else:
-                    print("   ⚠️ The provided generated text is empty after cleaning, cannot compute distribution error rate.") 
-
-        return results
+                else:
+                    print("   ⚠️ The provided generated text is empty after cleaning, cannot compute distribution error rate.") 
+
+        return results

{openstbench-1.2.0 → openstbench-1.2.3}/src/openstbench/translation_evaluator.py

@@ -1,24 +1,28 @@
-import os
 import gc
 import json
+import os
+from pathlib import Path
+from typing import Dict, List, Optional, Union
+
 import numpy as np
 import sacrebleu
 import torch
-from typing import Dict, List, Optional, Union
-from pathlib import Path
 
 from ._model_loading import resolve_pretrained_source
 
+
 # ==================== Configuration ====================
 
 CACHE_PATHS = {
     "huggingface": os.path.expanduser("~/.cache/huggingface/hub"),
 }
 
+
 # ==================== Optional Dependencies ====================
 
 try:
     from bleurt_pytorch import BleurtConfig, BleurtForSequenceClassification, BleurtTokenizer
+
     HAS_BLEURT = True
 except ImportError:
     HAS_BLEURT = False
@@ -29,46 +33,50 @@ except ImportError:
     download_model = None
     load_from_checkpoint = None
 
+
 # ==================== Input Loaders ====================
 
 def load_text_from_file_or_list(input_data: Union[str, List[str]], name: str = "text") -> List[str]:
 
     if isinstance(input_data, list):
         return input_data
-    
+
     if not isinstance(input_data, str):
         raise ValueError(f"{name} must be a file path (str) or a list of strings (List[str])")
 
     path = Path(input_data)
     if not path.exists():
         raise FileNotFoundError(f"{name} file not found: {input_data}")
-    
+
     suffix = path.suffix.lower()
-    
+
     if suffix == ".json":
         with open(path, "r", encoding="utf-8") as f:
             data = json.load(f)
-        
+
         if isinstance(data, list):
-            if len(data) == 0: return []
-            if isinstance(data[0], str): return data
+            if len(data) == 0:
+                return []
+            if isinstance(data[0], str):
+                return data
             if isinstance(data[0], dict):
                 for key in ["target_text", "hypothesis", "text", "ref", "reference", "src", "source"]:
                     if key in data[0]:
                         return [item[key] for item in data]
-                raise ValueError(f"JSON list items do not contain common text fields")
+                raise ValueError("JSON list items do not contain common text fields")
 
         if isinstance(data, dict):
             for key in ["target_text", "hypothesis", "text", "ref", "reference", "src", "source"]:
                 if key in data:
                     return data[key]
             raise ValueError("JSON dictionary does not contain common text fields")
-            
+
         raise ValueError("Unsupported JSON format")
     else:
         with open(path, "r", encoding="utf-8") as f:
             return [line.strip() for line in f if line.strip()]
 
+
 def load_audio_from_folder(folder_path: str, extensions: tuple = (".wav", ".mp3", ".flac")) -> List[str]:
     folder = Path(folder_path)
     if not folder.exists():
@@ -85,23 +93,31 @@ def load_audio_from_folder(folder_path: str, extensions: tuple = (".wav", ".mp3"
 # ==================== Evaluator Core Class ====================
 
 DEFAULT_COMET_MODEL = "Unbabel/wmt22-comet-da"
+DEFAULT_COMET_QE_MODEL = "Unbabel/wmt22-cometkiwi-da"
 DEFAULT_BLEURT_MODEL = "lucadiliello/BLEURT-20"
+COMET_METRIC_NAME = "COMET"
+COMET_QE_METRIC_NAME = "COMETKiwi"
+
 
 class TranslationEvaluator:
     """
-    Text-side Translation Quality Evaluator: Supports Text Translation Extraction and Comparison (BLEU, COMET, BLEURT...)
+    Text-side Translation Quality Evaluator: Supports text translation metrics
+    such as BLEU, chrF++, COMET, COMETKiwi, and BLEURT.
     """
 
-    def __init__(self, 
-                 use_bleu: bool = True,
-                 use_chrf: bool = True,
-                 use_comet: bool = True,      
-                 use_bleurt: bool = False,    
-                 comet_model: str = DEFAULT_COMET_MODEL,
-                 bleurt_path: Optional[str] = None,
-                 bleurt_model: Optional[str] = None,
-                 device: Optional[str] = None):
-        
+    def __init__(
+        self,
+        use_bleu: bool = True,
+        use_chrf: bool = True,
+        use_comet: bool = True,
+        use_bleurt: bool = False,
+        comet_model: str = DEFAULT_COMET_MODEL,
+        comet_qe_model: str = DEFAULT_COMET_QE_MODEL,
+        bleurt_path: Optional[str] = None,
+        bleurt_model: Optional[str] = None,
+        device: Optional[str] = None,
+    ):
+
         self.use_bleu = use_bleu
         self.use_chrf = use_chrf
         self.use_comet = use_comet
@@ -113,28 +129,30 @@ class TranslationEvaluator:
             self.device = "cuda"
         else:
             self.device = "cpu"
-        
-        print(f"🚀 Initializing Translation Evaluator (Text-only) on {self.device}...")
 
-        # Load language models
+        print(f"Initializing Translation Evaluator (Text-only) on {self.device}...")
+
+        self.comet_model = comet_model
+        self.comet_qe_model = comet_qe_model
         self.comet = None
-        if self.use_comet:
-            self.comet = self._load_comet(comet_model)
+        self.comet_qe = None
 
         self.bleurt_model = None
         self.bleurt_tokenizer = None
         if self.use_bleurt:
             self._load_bleurt(bleurt_path, bleurt_model)
 
-        print("✅ Translation text evaluation metrics system is ready!")
+        print("Translation text evaluation metrics system is ready!")
+
+    def __enter__(self):
+        return self
 
-    def __enter__(self): return self
-    def __exit__(self, exc_type, exc_val, exc_tb): 
+    def __exit__(self, exc_type, exc_val, exc_tb):
         self.cleanup()
         return False
 
     def cleanup(self):
-        for attr in ['comet', 'bleurt_model', 'bleurt_tokenizer']:
+        for attr in ["comet", "comet_qe", "bleurt_model", "bleurt_tokenizer"]:
             if hasattr(self, attr) and getattr(self, attr) is not None:
                 delattr(self, attr)
         if torch.cuda.is_available():
@@ -158,38 +176,38 @@ class TranslationEvaluator:
                 return str(ckpt.resolve())
         return None
 
-    def _load_comet(self, model_name: str):
+    def _load_comet(self, model_name: str, *, fallback_model: str, metric_name: str):
         if not download_model:
-            print("⚠️ COMET is not installed, skipping...")
+            print("COMET is not installed, skipping...")
             return None
         try:
             model_source, source_kind = resolve_pretrained_source(
                 model_name,
-                fallback_source=DEFAULT_COMET_MODEL,
+                fallback_source=fallback_model,
             )
             local_ckpt = self._resolve_local_comet_checkpoint(model_source)
             if local_ckpt is not None:
-                print(f"⏳ [Local] Loading COMET: {local_ckpt}")
+                print(f"[Local] Loading {metric_name}: {local_ckpt}")
                 model = load_from_checkpoint(local_ckpt)
             else:
-                remote_source = model_source if source_kind == "remote" else DEFAULT_COMET_MODEL
+                remote_source = model_source if source_kind == "remote" else fallback_model
                 cache = os.path.join(CACHE_PATHS["huggingface"], f"models--{remote_source.replace('/', '--')}")
                 status = "[Local]" if os.path.exists(cache) else "[Online]"
-                print(f"⏳ {status} Loading COMET: {model_name}")
-                print(f"Loading COMET ({status}) from {remote_source}")
+                print(f"{status} Loading {metric_name}: {model_name}")
+                print(f"Loading {metric_name} ({status}) from {remote_source}")
                 model = load_from_checkpoint(download_model(remote_source))
             if self.device.startswith("cuda"):
                 model = model.to(self.device)
             return model
         except Exception as e:
-            print(f"❌ COMET loading failed: {e}")
+            print(f"{metric_name} loading failed: {e}")
             return None
 
     def _load_bleurt(self, path: Optional[str], model_name: Optional[str]):
         if not HAS_BLEURT:
-            print("⚠️ bleurt-pytorch is not installed, skipping...")
+            print("bleurt-pytorch is not installed, skipping...")
             return
-        
+
         if path:
             model_source, _source_kind = resolve_pretrained_source(
                 path,
@@ -197,30 +215,40 @@ class TranslationEvaluator:
             )
         else:
             model_source, _source_kind = resolve_pretrained_source(model_name or DEFAULT_BLEURT_MODEL)
-        print(f"⏳ Loading BLEURT: {model_source}")
-        
+        print(f"Loading BLEURT: {model_source}")
+
         try:
             self.bleurt_tokenizer = BleurtTokenizer.from_pretrained(model_source)
             self.bleurt_model = BleurtForSequenceClassification.from_pretrained(model_source)
             self.bleurt_model = self.bleurt_model.to(self.device).eval()
         except Exception as e:
-            print(f"❌ BLEURT loading failed: {e}")
-
+            print(f"BLEURT loading failed: {e}")
 
     def _get_bleu_tokenizer_name(self, lang: str) -> str:
-        if lang == 'zh': return 'zh'
-        elif lang == 'ja': return 'ja-mecab'
-        elif lang == 'ko': return 'ko-mecab'
-        else: return '13a'
+        if lang == "zh":
+            return "zh"
+        elif lang == "ja":
+            return "ja-mecab"
+        elif lang == "ko":
+            return "ko-mecab"
+        else:
+            return "13a"
 
     def _compute_bleurt_score(self, references: List[str], candidates: List[str]) -> float:
         all_scores = []
         batch_size = 32
         for i in range(0, len(references), batch_size):
-            br = references[i:i+batch_size]
-            bc = candidates[i:i+batch_size]
+            br = references[i : i + batch_size]
+            bc = candidates[i : i + batch_size]
             with torch.no_grad():
-                inputs = self.bleurt_tokenizer(br, bc, padding='longest', truncation=True, max_length=512, return_tensors='pt')
+                inputs = self.bleurt_tokenizer(
+                    br,
+                    bc,
+                    padding="longest",
+                    truncation=True,
+                    max_length=512,
+                    return_tensors="pt",
+                )
                 inputs = {k: v.to(self.device) for k, v in inputs.items()}
                 scores = self.bleurt_model(**inputs).logits.flatten().tolist()
                 all_scores.extend(scores)
@@ -228,61 +256,95 @@ class TranslationEvaluator:
 
     def evaluate_all(
         self,
-        reference: Union[List[str], str],
-        target_text: Union[List[str], str],
+        reference: Optional[Union[List[str], str]] = None,
+        target_text: Optional[Union[List[str], str]] = None,
         source: Optional[Union[List[str], str]] = None,
-        target_lang: str = "en"
+        target_lang: str = "en",
     ) -> Dict[str, float]:
         """
         Args:
-            reference: Ground truth benchmark translation reference from the dataset (List or file path).
-            target_text: Direct text translation output from the translation model (List or file path).
-            source: Optional original source text (mandatory for COMET).
-            target_lang: Target language, affects BLEU Tokenizer selection.
+            reference: Optional benchmark translation reference from the dataset.
+            target_text: Direct text translation output from the translation model.
+            source: Optional original source text, mandatory for COMET and COMETKiwi.
+            target_lang: Target language, affects BLEU tokenizer selection.
         """
         results = {}
         print(f"\n--- Starting Text Translation Quality Evaluation (Target Lang: {target_lang}) ---")
 
-        final_ref = load_text_from_file_or_list(reference, "Reference")
+        if target_text is None:
+            raise ValueError("target_text is required")
         final_text = load_text_from_file_or_list(target_text, "Target Text")
-        
+
+        final_ref = None
+        if reference is not None:
+            loaded_ref = load_text_from_file_or_list(reference, "Reference")
+            final_ref = loaded_ref if loaded_ref else None
+
         final_src = None
         if source:
             final_src = load_text_from_file_or_list(source, "Source")
-            if len(final_src) != len(final_ref):
-                raise ValueError("Source and Reference have different lengths")
-        
-        if len(final_text) != len(final_ref):
+            if len(final_src) != len(final_text):
+                raise ValueError("Source and Target Text have different lengths")
+
+        if final_ref is not None and len(final_text) != len(final_ref):
             raise ValueError("Target Text and Reference have different lengths")
 
         # 1. sacreBLEU
-        if self.use_bleu:
+        if self.use_bleu and final_ref is not None:
             tokenizer_name = self._get_bleu_tokenizer_name(target_lang)
             try:
                 results["sacreBLEU"] = sacrebleu.corpus_bleu(final_text, [final_ref], tokenize=tokenizer_name).score
-            except Exception as e:
+            except Exception:
                 results["sacreBLEU"] = -1.0
 
         # 2. chrF++
-        if self.use_chrf:
+        if self.use_chrf and final_ref is not None:
             try:
                 results["chrF++"] = sacrebleu.corpus_chrf(final_text, [final_ref], word_order=2).score
-            except: results["chrF++"] = -1.0
+            except Exception:
+                results["chrF++"] = -1.0
 
         # 3. BLEURT
-        if self.use_bleurt and self.bleurt_model:
+        if self.use_bleurt and self.bleurt_model and final_ref is not None:
             try:
                 results["BLEURT"] = self._compute_bleurt_score(final_ref, final_text)
-            except Exception as e:
+            except Exception:
                 results["BLEURT"] = -1.0
 
-        # 4. COMET
-        if self.use_comet and self.comet and final_src:
-            try:
-                data = [{"src": s, "mt": t, "ref": r} for s, t, r in zip(final_src, final_text, final_ref)]
+        # 4. COMET and COMETKiwi.
+        if self.use_comet:
+            if not final_src:
+                if final_ref is not None:
+                    results[COMET_METRIC_NAME] = -1.0
+                results[COMET_QE_METRIC_NAME] = -1.0
+            else:
                 gpus = 1 if self.device.startswith("cuda") else 0
-                results["COMET"] = self.comet.predict(data, batch_size=8, gpus=gpus).system_score
-            except Exception as e:
-                results["COMET"] = -1.0
-        
+
+                if final_ref is not None:
+                    if self.comet is None:
+                        self.comet = self._load_comet(
+                            self.comet_model,
+                            fallback_model=DEFAULT_COMET_MODEL,
+                            metric_name=COMET_METRIC_NAME,
+                        )
+                    if self.comet:
+                        try:
+                            data = [{"src": s, "mt": t, "ref": r} for s, t, r in zip(final_src, final_text, final_ref)]
+                            results[COMET_METRIC_NAME] = self.comet.predict(data, batch_size=8, gpus=gpus).system_score
+                        except Exception:
+                            results[COMET_METRIC_NAME] = -1.0
+
+                if self.comet_qe is None:
+                    self.comet_qe = self._load_comet(
+                        self.comet_qe_model,
+                        fallback_model=DEFAULT_COMET_QE_MODEL,
+                        metric_name=COMET_QE_METRIC_NAME,
+                    )
+                if self.comet_qe:
+                    try:
+                        data = [{"src": s, "mt": t} for s, t in zip(final_src, final_text)]
+                        results[COMET_QE_METRIC_NAME] = self.comet_qe.predict(data, batch_size=8, gpus=gpus).system_score
+                    except Exception:
+                        results[COMET_QE_METRIC_NAME] = -1.0
+
         return {k: round(v, 4) if v >= 0 else v for k, v in results.items()}

openstbench-1.2.0/src/openstbench/latency/__init__.py

@@ -1,14 +0,0 @@
-# This file contains code adapted from SimulEval:
-# https://github.com/facebookresearch/SimulEval
-#
-# SimulEval is licensed under the Creative Commons Attribution-ShareAlike
-# 4.0 International License (CC BY-SA 4.0):
-# https://creativecommons.org/licenses/by-sa/4.0/
-#
-# The adapted portions in this file are distributed under CC BY-SA 4.0.
-# Modifications were made for OpenSTBench.
-
-from .agent import GenericAgent, AgentPipeline
-from .basics import ReadAction, WriteAction, TextSegment, SpeechSegment
-from .cli import LatencyEvaluator
-from .metrics import register # Allows users to register custom Scorers