OpenSTBench 0.2.0__py3-none-any.whl

openstbench/__init__.py

@@ -0,0 +1,143 @@
+"""Top-level exports for the ``openstbench`` package.
+
+This module keeps package import resilient to optional dependency failures.
+Importing ``openstbench`` should succeed even when a specific evaluator's
+extra dependencies are missing; the failure is raised only when that symbol is
+actually accessed.
+"""
+
+from importlib import import_module
+from typing import Dict, Tuple
+
+__version__ = "0.2.0"
+
+__all__ = [
+    "TranslationEvaluator",
+    "EmotionEvaluator",
+    "ParalinguisticEvaluator",
+    "BaseAudioEventPredictor",
+    "BaseAudioEventLocalizer",
+    "ClapAudioEventPredictor",
+    "ClapSlidingWindowEventLocalizer",
+    "EventAlignmentConfig",
+    "EventLocalization",
+    "EventLocalizationConfig",
+    "EventPrediction",
+    "EventPredictionConfig",
+    "ParalinguisticSample",
+    "load_paralinguistic_manifest",
+    "load_paralinguistic_samples",
+    "load_paralinguistic_audio_from_folder",
+    "build_paralinguistic_inputs",
+    "evaluate_paralinguistic_dataset",
+    "SpeechQualityEvaluator",
+    "LatencyEvaluator",
+    "SpeakerSimilarityEvaluator",
+    "GenericAgent",
+    "AgentPipeline",
+    "ReadAction",
+    "WriteAction",
+    "load_text_from_file_or_list",
+    "load_audio_from_folder",
+    "Dataset",
+    "load_dataset",
+    "list_datasets",
+    "get_dataset_info",
+    "create_dataset_from_json",
+]
+
+
+_EXPORT_SPECS: Tuple[Tuple[str, Tuple[str, ...]], ...] = (
+    (
+        "translation_evaluator",
+        (
+            "TranslationEvaluator",
+            "load_audio_from_folder",
+            "load_text_from_file_or_list",
+        ),
+    ),
+    ("emotion_evaluator", ("EmotionEvaluator",)),
+    (
+        "paralinguistic_evaluator",
+        (
+            "BaseAudioEventPredictor",
+            "BaseAudioEventLocalizer",
+            "ClapAudioEventPredictor",
+            "ClapSlidingWindowEventLocalizer",
+            "EventAlignmentConfig",
+            "EventLocalization",
+            "EventLocalizationConfig",
+            "EventPrediction",
+            "EventPredictionConfig",
+            "ParalinguisticEvaluator",
+            "ParalinguisticSample",
+            "build_paralinguistic_inputs",
+            "evaluate_paralinguistic_dataset",
+            "load_audio_from_folder",
+            "load_paralinguistic_manifest",
+            "load_paralinguistic_samples",
+        ),
+    ),
+    ("speech_quality_evaluator", ("SpeechQualityEvaluator",)),
+    ("speaker_similarity_evaluator", ("SpeakerSimilarityEvaluator",)),
+    (
+        "dataset",
+        (
+            "Dataset",
+            "create_dataset_from_json",
+            "get_dataset_info",
+            "list_datasets",
+            "load_dataset",
+        ),
+    ),
+    ("latency.agent", ("AgentPipeline", "GenericAgent")),
+    ("latency.basics", ("ReadAction", "WriteAction")),
+    ("latency.cli", ("LatencyEvaluator",)),
+)
+
+
+_IMPORT_ERRORS: Dict[str, Tuple[str, Exception]] = {}
+_SYMBOL_TO_MODULE: Dict[str, str] = {}
+
+for _module_name, _names in _EXPORT_SPECS:
+    for _name in _names:
+        if _module_name == "paralinguistic_evaluator" and _name == "load_audio_from_folder":
+            _SYMBOL_TO_MODULE["load_paralinguistic_audio_from_folder"] = _module_name
+        else:
+            _SYMBOL_TO_MODULE[_name] = _module_name
+
+
+def _load_module_exports(module_name: str) -> None:
+    names = [name for name, owner in _SYMBOL_TO_MODULE.items() if owner == module_name]
+    try:
+        module = import_module(f".{module_name}", __name__)
+    except Exception as exc:  # pragma: no cover - depends on optional deps
+        for name in names:
+            _IMPORT_ERRORS[name] = (module_name, exc)
+        return
+
+    for name in names:
+        if module_name == "paralinguistic_evaluator" and name == "load_paralinguistic_audio_from_folder":
+            globals()[name] = getattr(module, "load_audio_from_folder")
+        else:
+            globals()[name] = getattr(module, name)
+        _IMPORT_ERRORS.pop(name, None)
+
+
+def __getattr__(name: str):
+    if name in _SYMBOL_TO_MODULE:
+        module_name = _SYMBOL_TO_MODULE[name]
+        _load_module_exports(module_name)
+        if name in globals():
+            return globals()[name]
+    if name in _IMPORT_ERRORS:
+        module_name, exc = _IMPORT_ERRORS[name]
+        raise ImportError(
+            f"Cannot import '{name}' from 'openstbench' because "
+            f"'openstbench.{module_name}' failed to load: {exc}"
+        ) from exc
+    raise AttributeError(f"module '{__name__}' has no attribute '{name}'")
+
+
+def __dir__():
+    return sorted(set(globals()) | set(__all__))

openstbench/dataset.py

@@ -0,0 +1,199 @@
+"""
+内置数据集管理 - 支持音频文件下载
+优先使用本地缓存，无则联网下载
+"""
+import os
+import json
+import urllib.request
+import zipfile
+import shutil
+from typing import Dict, List, Optional, Union
+
+# 数据集下载地址
+DATASET_URLS = {
+    "zh-en-littleprince": "https://github.com/sjtuayj/OpenSTBench/releases/download/v0.1.0/zh-en-littleprince.zip",
+}
+
+# 默认缓存目录
+# DEFAULT_CACHE_DIR = os.path.expanduser("~/.datasets")
+DEFAULT_CACHE_DIR = "./datasets"
+
+class Dataset:
+    """内置数据集类"""
+    
+    def __init__(self, data: List[Dict], base_dir: str):
+        self._data = data
+        self._base_dir = base_dir
+    
+    def __len__(self) -> int:
+        return len(self._data)
+    
+    def __getitem__(self, idx: int) -> Dict:
+        item = self._data[idx].copy()
+        if "source_speech_path" in item:
+            filename = os.path.basename(item["source_speech_path"])
+            item["source_speech_path"] = os.path.join(self._base_dir, "audio", filename)
+        return item
+    
+    @property
+    def ids(self) -> List[str]:
+        return [item.get("id", f"sample_{i}") for i, item in enumerate(self._data)]
+    
+    @property
+    def source_texts(self) -> List[str]:
+        return [item["source_text"] for item in self._data]
+    
+    @property
+    def reference_texts(self) -> List[str]:
+        return [item["reference_text"] for item in self._data]
+    
+    @property
+    def audio_paths(self) -> List[str]:
+        return [self[i].get("source_speech_path", "") for i in range(len(self))]
+    
+    def verify_audio_files(self) -> Dict[str, Union[int, List[str]]]:
+        """验证音频文件完整性"""
+        missing = [p for p in self.audio_paths if not os.path.exists(p)]
+        return {
+            "total": len(self),
+            "found": len(self) - len(missing),
+            "missing": len(missing),
+            "missing_files": missing,
+        }
+
+
+def list_datasets() -> List[str]:
+    """列出所有可用数据集"""
+    return list(DATASET_URLS.keys())
+
+
+def _is_dataset_cached(name: str, cache_dir: str) -> bool:
+    """检查数据集是否已下载"""
+    dataset_dir = os.path.join(cache_dir, name)
+    data_file = os.path.join(dataset_dir, "dataset_paired.json")
+    audio_dir = os.path.join(dataset_dir, "audio")
+    return os.path.exists(data_file) and os.path.exists(audio_dir)
+
+
+def get_dataset_info(name: str, cache_dir: Optional[str] = None) -> Dict:
+    """获取数据集信息"""
+    if name not in DATASET_URLS:
+        raise ValueError(f"未知数据集: {name}")
+    
+    cache_dir = cache_dir or DEFAULT_CACHE_DIR
+    is_cached = _is_dataset_cached(name, cache_dir)
+    
+    info = {
+        "name": name,
+        "url": DATASET_URLS[name],
+        "cache_dir": os.path.join(cache_dir, name),
+        "is_downloaded": is_cached,
+    }
+    
+    if is_cached:
+        dataset = load_dataset(name, cache_dir=cache_dir)
+        info["num_samples"] = len(dataset)
+        verify = dataset.verify_audio_files()
+        info["audio_complete"] = verify["missing"] == 0
+    
+    return info
+
+
+def load_dataset(
+    name: str,
+    cache_dir: Optional[str] = None,
+    force_download: bool = False,
+) -> Dataset:
+    """
+    加载数据集（优先本地，无则下载）
+    
+    Args:
+        name: 数据集名称
+        cache_dir: 缓存目录
+        force_download: 强制重新下载
+    
+    Returns:
+        Dataset 对象
+    """
+    if name not in DATASET_URLS:
+        available = ", ".join(DATASET_URLS.keys())
+        raise ValueError(f"未知数据集: {name}。可用: {available}")
+    
+    cache_dir = cache_dir or DEFAULT_CACHE_DIR
+    dataset_dir = os.path.join(cache_dir, name)
+    data_file = os.path.join(dataset_dir, "dataset_paired.json")
+    
+    # 检查本地缓存
+    if _is_dataset_cached(name, cache_dir) and not force_download:
+        print(f"✅ [Local] 使用本地数据集: {name}")
+        print(f"   路径: {dataset_dir}")
+    else:
+        print(f"⏳ [Online] 下载数据集: {name}")
+        _download_dataset(name, cache_dir)
+    
+    # 加载数据
+    with open(data_file, "r", encoding="utf-8") as f:
+        data = json.load(f)
+    
+    dataset = Dataset(data=data, base_dir=dataset_dir)
+    
+    # 验证完整性
+    verify = dataset.verify_audio_files()
+    if verify["missing"] > 0:
+        print(f"   ⚠️ 缺少 {verify['missing']} 个音频文件")
+    else:
+        print(f"   ✅ 数据完整 ({verify['total']} 条样本, 音频齐全)")
+    
+    return dataset
+
+
+def _download_dataset(name: str, cache_dir: str):
+    """下载数据集"""
+    url = DATASET_URLS[name]
+    dataset_dir = os.path.join(cache_dir, name)
+    zip_path = os.path.join(cache_dir, f"{name}.zip")
+    
+    os.makedirs(cache_dir, exist_ok=True)
+    
+    if os.path.exists(dataset_dir):
+        shutil.rmtree(dataset_dir)
+    
+    print(f"   URL: {url}")
+    
+    try:
+        urllib.request.urlretrieve(url, zip_path, _download_progress)
+        print()
+        
+        print(f"   📦 解压中...")
+        os.makedirs(dataset_dir, exist_ok=True)
+        with zipfile.ZipFile(zip_path, 'r') as zf:
+            zf.extractall(dataset_dir)
+        
+        os.remove(zip_path)
+        print(f"   ✅ 下载完成: {dataset_dir}")
+        
+    except Exception as e:
+        if os.path.exists(zip_path):
+            os.remove(zip_path)
+        if os.path.exists(dataset_dir):
+            shutil.rmtree(dataset_dir)
+        raise RuntimeError(f"下载失败: {e}")
+
+
+def _download_progress(block_num, block_size, total_size):
+    """下载进度条"""
+    downloaded = block_num * block_size
+    if total_size > 0:
+        percent = min(100, downloaded * 100 // total_size)
+        bar_len = 40
+        filled = int(bar_len * percent // 100)
+        bar = "█" * filled + "░" * (bar_len - filled)
+        print(f"\r   [{bar}] {percent}%", end="", flush=True)
+
+
+def create_dataset_from_json(json_path: str) -> Dataset:
+    """从本地 JSON 创建数据集"""
+    with open(json_path, "r", encoding="utf-8") as f:
+        data = json.load(f)
+    base_dir = os.path.dirname(os.path.abspath(json_path))
+    return Dataset(data=data, base_dir=base_dir)

openstbench/emotion_evaluator.py

@@ -0,0 +1,262 @@
+import os
+import json
+import torch
+import numpy as np
+from pathlib import Path
+from typing import List, Dict, Optional, Union
+from tqdm import tqdm
+from numpy.linalg import norm
+
+def _load_data_list(
+    input_data: Union[str, List[str]], 
+    name: str,
+    target_type: str = "audio"
+) -> List[str]:
+    if isinstance(input_data, list):
+        return input_data
+    if not isinstance(input_data, str):
+        raise ValueError(f"{name} 必须是 文件路径(str) 或 列表(List[str])")
+
+    path = Path(input_data)
+    if not path.exists():
+        raise FileNotFoundError(f"{name} 文件不存在: {input_data}")
+    
+    suffix = path.suffix.lower()
+
+    if suffix == ".json":
+        with open(path, "r", encoding="utf-8") as f:
+            data = json.load(f)
+        
+        if target_type == "audio":
+            candidate_keys = ["audio", "path", "file", "wav", "mp3"]
+        elif target_type == "label":
+            candidate_keys = ["label", "emotion", "class", "reference"]
+        else:
+            candidate_keys = ["text", "sentence", "content", "transcript"]
+
+        if isinstance(data, list):
+            if not data: return []
+            if isinstance(data[0], str): return data
+            if isinstance(data[0], dict):
+                for key in candidate_keys:
+                    if key in data[0]:
+                        return [item[key] for item in data]
+                raise ValueError(f"JSON 列表项中未找到常见{target_type}字段: {candidate_keys}")
+
+        if isinstance(data, dict):
+            plural_candidates = [k + "s" for k in candidate_keys]
+            plural_candidates += ["target_text", "hypothesis", "source_text"] if target_type == "text" else []
+            for key in plural_candidates:
+                if key in data:
+                    return data[key]
+            raise ValueError(f"JSON 字典中未找到常见{target_type}列表字段")
+        raise ValueError("不支持的 JSON 格式")
+    else:
+        with open(path, "r", encoding="utf-8") as f:
+            return [line.strip() for line in f if line.strip()]
+
+
+def _load_audio_from_folder(folder_path: str, extensions: tuple = (".wav", ".mp3", ".flac")) -> List[str]:
+    folder = Path(folder_path)
+    if not folder.exists():
+        raise FileNotFoundError(f"文件夹不存在: {folder_path}")
+    audio_files = []
+    for ext in extensions:
+        audio_files.extend(folder.glob(f"*{ext}"))
+    audio_files = sorted(audio_files, key=lambda x: x.stem)
+    if not audio_files:
+        raise ValueError(f"文件夹中没有音频文件: {folder_path}")
+    return [str(f) for f in audio_files]
+
+
+class EmotionEvaluator:
+    """
+    基于 Emotion2Vec+ 的情感评测器 
+    支持：
+    [1] 跨语种保真度评测：基于 768-d 高维嵌入特征的 Cosine Similarity 保真度。
+    [2] 离散情感分类评测：基于模型 Zero-Shot 分类能力的识别准确度评测。
+    """
+    
+    # 唯一依赖的基座模型
+    DEFAULT_E2V_MODEL = "iic/emotion2vec_plus_large"
+
+    def __init__(self, 
+                 e2v_model_path: Optional[str] = None,
+                 custom_label_map: Optional[Dict[str, str]] = None,
+                 device: Optional[str] = None):
+        
+        self.device = device if device else ("cuda" if torch.cuda.is_available() else "cpu")
+        self.e2v_model_path = e2v_model_path or self.DEFAULT_E2V_MODEL
+        self.e2v_model = None
+        
+        # 将用户的自定义标签映射统一全部小写，确保健壮性
+        self.custom_label_map = {k.lower(): v.lower() for k, v in (custom_label_map or {}).items()}
+
+    def _load_e2v_model(self):
+        if self.e2v_model is not None: return
+        print(f"⏳ 正在加载 Emotion2Vec+ 大模型: {self.e2v_model_path}")
+        try:
+            from funasr import AutoModel
+            self.e2v_model = AutoModel(model=self.e2v_model_path, device=self.device, disable_update=True)
+            print("✅ Emotion2Vec+ 大模型加载成功！")
+        except ImportError:
+            print("❌ Emotion2Vec+ 依赖缺失。请执行: pip install funasr modelscope")
+        except Exception as e:
+            print(f"❌ Emotion2Vec+ 分类模型加载失败: {e}")
+
+    # =============== 特征提取部分 ===============
+
+    def _extract_e2v_embeddings(self, audio_paths: List[str]) -> List[np.ndarray]:
+        """提取 Emotion2Vec+ 的综合高维特征 Embedding 用于衡量总体保真度相似度"""
+        self._load_e2v_model()
+        if not self.e2v_model: return [np.zeros(768)] * len(audio_paths)
+
+        results = []
+        for path in tqdm(audio_paths, desc="🔍 音频 E2V 高维特征", unit="file"):
+            try:
+                res = self.e2v_model.generate(path, output_dir=None, granularity="utterance", extract_embedding=True)
+                if isinstance(res, list) and len(res) > 0 and 'feats' in res[0]:
+                    emb = np.array(res[0]['feats']).squeeze()
+                    results.append(emb)
+                else:
+                    results.append(np.zeros(768))
+            except Exception as e:
+                results.append(np.zeros(768))
+        return results
+
+    def _extract_cls_emotion(self, audio_paths: List[str]) -> List[str]:
+        """提取 Emotion2Vec+ 的离散情感分类预测结果"""
+        self._load_e2v_model()
+        if not self.e2v_model: return ["unknown"] * len(audio_paths)
+
+        results = []
+        for path in tqdm(audio_paths, desc="🔍 音频 E2V 离散分类标签 예측", unit="file"):
+            try:
+                # 默认分类模式，返回具体的 label
+                res = self.e2v_model.generate(path, output_dir=None, granularity="utterance", extract_embedding=False)
+                if isinstance(res, list) and len(res) > 0 and 'labels' in res[0] and 'scores' in res[0]:
+                    labels = res[0]['labels']
+                    scores = res[0]['scores']
+                    
+                    # 1. 找到得分最高的标签索引
+                    best_idx = np.argmax(scores)
+                    best_label_str = labels[best_idx]
+                    
+                    # 2. 解析出英文部分 (因为输出格式是 '生气/angry')
+                    if "/" in best_label_str:
+                        label = best_label_str.split("/")[-1].lower()
+                    else:
+                        label = best_label_str.lower()
+                        
+                    # 3. 清洗可能存在的特殊占位符
+                    label = label.replace("<|", "").replace("|>", "").strip()
+                else:
+                    label = "unknown"
+                
+                # 应用自定义对齐
+                if label in self.custom_label_map:
+                    label = self.custom_label_map[label]
+                results.append(label)
+            except Exception as e:
+                results.append("unknown")
+        return results
+
+    # =============== 评测主入口 ===============
+
+    def evaluate_all(self, 
+                     source_audio: Optional[Union[List[str], str]] = None, 
+                     target_audio: Optional[Union[List[str], str]] = None,
+                     reference_labels: Optional[Union[List[str], str]] = None,
+                     verbose: bool = True) -> Dict[str, float]:
+        """
+        动态融合引擎入口：
+        - 仅传 target_audio 与 reference_labels -> 计算 Accuracy 
+        - 仅传 source_audio 与 target_audio -> 计算 E2V 特征保真度
+        """
+        
+        if target_audio is None:
+            if source_audio is not None and reference_labels is not None:
+                target_audio = source_audio
+                source_audio = None
+            else:
+                raise ValueError("🚨 必须提供 target_audio 参数运行评测。")
+
+        if isinstance(target_audio, str) and os.path.isdir(target_audio):
+            tgt_paths = _load_audio_from_folder(target_audio)
+        else:
+            tgt_paths = _load_data_list(target_audio, "Target Audio Paths", "audio")
+        
+        num_samples = len(tgt_paths)
+        if num_samples == 0:
+            raise ValueError("没有找到目标音频数据，评测停止。")
+
+        run_fidelity = source_audio is not None
+        run_classification = reference_labels is not None
+        results = {}
+
+        # ==================== 分支 1：保真度相似度计算 ====================
+        if run_fidelity:
+            if verbose: print(f"\n📝 启动【跨模态综合保真度运算】 ({num_samples} 个 Source-Target 音频距离比对)...")
+            
+            if isinstance(source_audio, str) and os.path.isdir(source_audio):
+                src_paths = _load_audio_from_folder(source_audio)
+            else:
+                src_paths = _load_data_list(source_audio, "Source Audio Paths", "audio")
+
+            if len(src_paths) != num_samples:
+                raise ValueError(f"数目不一致: Source ({len(src_paths)}) != Target ({num_samples})")
+
+            src_e2v_embs = self._extract_e2v_embeddings(src_paths)
+            tgt_e2v_embs = self._extract_e2v_embeddings(tgt_paths)
+
+            e2v_cosine_sim_total = 0.0 
+            valid_e2v_count = 0
+            
+            for i in range(num_samples):
+                s_emb = src_e2v_embs[i]
+                t_emb = tgt_e2v_embs[i]
+                
+                n_s = norm(s_emb)
+                n_t = norm(t_emb)
+                if n_s > 0 and n_t > 0:
+                    sim = np.dot(s_emb, t_emb) / (n_s * n_t)
+                    e2v_cosine_sim_total += float(sim)
+                    valid_e2v_count += 1
+
+            final_cosine = (e2v_cosine_sim_total / valid_e2v_count) if valid_e2v_count > 0 else 0.0
+            results["Emotion2Vec_Cosine_Similarity"] = round(final_cosine, 4)
+
+
+        # ==================== 分支 2：离散情感识别计算 ====================
+        if run_classification:
+            if verbose: print(f"\n📝 启动【Emotion2Vec+ 离散分类准确率运算】 ({num_samples} 个特征识别与金标准比对)...")
+            refs = _load_data_list(reference_labels, "Reference Labels", "label")
+            
+            if len(refs) != num_samples:
+                raise ValueError(f"参考标签数量 ({len(refs)}) 与目标音频数量 ({num_samples}) 不匹配！")
+
+            preds = self._extract_cls_emotion(tgt_paths)
+            
+            correct = 0
+            for p, r in zip(preds, refs):
+                if p.strip() == r.strip().lower():
+                    correct += 1
+            
+            acc = correct / len(refs) if len(refs) > 0 else 0.0
+            results["Audio_Emotion_Accuracy"] = round(acc, 4)
+
+
+        # ==================== 输出反馈 ====================
+        if verbose:
+            print("\n📊 [EmotionEvaluator] Emotion2Vec+ 综合评测报告:")
+            print(f"   - 有效评测样本量: {num_samples}条")
+            
+            if run_fidelity:
+                print("\n   [深度学习综合特征保真度] (数值范围[-1, 1]，越接近 1.00 代表整体情感越相似)")
+                print(f"   - Emotion2Vec+ 综合情感特征余弦相似度:         {results['Emotion2Vec_Cosine_Similarity']:.4f}")
+            
+            if run_classification:
+                print("\n   [深度学习离散情感识别分类率] (数值越高代表预设标签识别越准)")
+                print(f"   - Audio Emotion Accuracy (离散情感识别准确率): {results['Audio_Emotion_Accuracy']:.2%}")
+                
+        return results

openstbench/latency/__init__.py

@@ -0,0 +1,4 @@
+from .agent import GenericAgent, AgentPipeline
+from .basics import ReadAction, WriteAction, TextSegment, SpeechSegment
+from .cli import LatencyEvaluator
+from .metrics import register # 方便用户注册自己的 Scorer