OpenSTBench 0.3.1__tar.gz → 0.3.2__tar.gz

{openstbench-0.3.1 → openstbench-0.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: OpenSTBench
-Version: 0.3.1
+Version: 0.3.2
 Summary: Open speech and translation benchmarking toolkit supporting MT, ASR, TTS, SimulST, VC, and paralinguistics with optimized CJK language support
 Author-email: Yanjie An <691476922@qq.com>
 License: MIT

{openstbench-0.3.1 → openstbench-0.3.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "OpenSTBench"
-version = "0.3.1"  
+version = "0.3.2"  
 description = "Open speech and translation benchmarking toolkit supporting MT, ASR, TTS, SimulST, VC, and paralinguistics with optimized CJK language support"
 readme = {file = "README.md", content-type = "text/markdown"}
 requires-python = ">=3.9,<3.11"

{openstbench-0.3.1 → openstbench-0.3.2}/src/OpenSTBench.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: OpenSTBench
-Version: 0.3.1
+Version: 0.3.2
 Summary: Open speech and translation benchmarking toolkit supporting MT, ASR, TTS, SimulST, VC, and paralinguistics with optimized CJK language support
 Author-email: Yanjie An <691476922@qq.com>
 License: MIT

{openstbench-0.3.1 → openstbench-0.3.2}/src/openstbench/__init__.py

@@ -9,7 +9,7 @@ actually accessed.
 from importlib import import_module
 from typing import Dict, Tuple
 
-__version__ = "0.3.1"
+__version__ = "0.3.2"
 
 __all__ = [
     "TranslationEvaluator",

{openstbench-0.3.1 → openstbench-0.3.2}/src/openstbench/latency/cli.py

@@ -1,25 +1,26 @@
 import argparse
 import importlib.util
 import sys
+import time
 from pathlib import Path
 
 from tqdm import tqdm
 
 from .agent import GenericAgent
-from .instance import SpeechToTextInstance, SpeechToSpeechInstance
+from .instance import SpeechToSpeechInstance, SpeechToTextInstance
 from .metrics import SCORERS
 from .utils import Visualizer, materialize_s2s_alignment_artifacts, submit_slurm
-
-# --- 引入兄弟模块 (紧耦合) ---
-# 注意：使用相对导入或包绝对导入
 from ..translation_evaluator import TranslationEvaluator
 
+
 class LatencyEvaluator:
-    """同传延迟与质量评测器"""
+    """Generic latency evaluator for speech-input streaming tasks."""
+
     def __init__(
         self,
         agent: GenericAgent,
         segment_size=20,
+        poll_interval_ms=10.0,
         latency_unit="word",
         asr_fallback_for_s2s_alignment=True,
         asr_model="medium",
@@ -29,6 +30,7 @@ class LatencyEvaluator:
     ):
         self.agent = agent
         self.segment_size = segment_size
+        self.poll_interval_ms = max(1.0, float(poll_interval_ms))
         self.instances = {}
         self.latency_unit = latency_unit
         self.asr_fallback_for_s2s_alignment = asr_fallback_for_s2s_alignment
@@ -40,33 +42,62 @@ class LatencyEvaluator:
     def run(self, source_files, ref_files, task="s2t", output_dir="./output", visualize=False):
         Path(output_dir).mkdir(parents=True, exist_ok=True)
         visualizer = Visualizer(output_dir) if visualize else None
-        
-        print(f"🚀 Running Latency Evaluation ({task}, {len(source_files)} samples)...")
+        poll_interval_s = self.poll_interval_ms / 1000.0
+        chunk_interval_s = float(self.segment_size) / 1000.0
+        self.instances = {}
+
+        print(f"Running Latency Evaluation ({task}, {len(source_files)} samples)...")
         for i, (src, ref) in tqdm(enumerate(zip(source_files, ref_files)), total=len(source_files)):
             if task == "s2t":
                 ins = SpeechToTextInstance(i, src, ref, output_dir)
             else:
                 ins = SpeechToSpeechInstance(i, src, ref, output_dir)
-            
+
             self.agent.reset()
+            sample_wall_start = time.perf_counter()
+            ins.start_time = sample_wall_start
+            send_start_time = None
+            sent_chunk_count = 0
+            source_finished = False
+
             while not ins.finish_prediction:
-                s_in = ins.send_source(self.segment_size)
-                import time
-                t0 = time.perf_counter()
-                s_out = self.agent.pushpop(s_in)
-                t1 = time.perf_counter()
-                ins.add_inference_time(t1 - t0)
+                now = time.monotonic()
+                next_due_time = None
+
+                if not source_finished:
+                    if send_start_time is None:
+                        send_start_time = now
+                    next_due_time = send_start_time + (sent_chunk_count + 1) * chunk_interval_s
+                    if now >= next_due_time:
+                        s_in = ins.send_source(self.segment_size)
+                        self.agent.push(s_in)
+                        if not getattr(s_in, "is_empty", False):
+                            sent_chunk_count += 1
+                        source_finished = bool(getattr(s_in, "finished", False))
+
+                s_out = self.agent.pop()
                 if hasattr(self.agent, "consume_model_inference_time"):
                     ins.add_model_inference_time(self.agent.consume_model_inference_time())
-                ins.receive_prediction(s_out)
-            
+
+                if not getattr(s_out, "is_empty", False) or getattr(s_out, "finished", False):
+                    ins.receive_prediction(s_out)
+                    continue
+
+                if source_finished:
+                    time.sleep(poll_interval_s)
+                elif next_due_time is not None:
+                    sleep_time = max(0.0, min(poll_interval_s, next_due_time - time.monotonic()))
+                    if sleep_time > 0:
+                        time.sleep(sleep_time)
+
+            ins.total_inference_time = time.perf_counter() - sample_wall_start
             self.instances[i] = ins
-            if visualizer: visualizer.plot(ins.summarize())
-        
-        # S2S 准备文本
+            if visualizer:
+                visualizer.plot(ins.summarize())
+
         if task == "s2s":
             self._prepare_s2s_transcripts(output_dir)
-            
+
         return self.instances
 
     def _prepare_s2s_transcripts(self, output_dir):
@@ -79,10 +110,8 @@ class LatencyEvaluator:
             asr_device=self.asr_device,
         )
 
-        # 增加 show_all_metrics 开关，默认为 False
     def compute_latency(self, computation_aware=False, output_dir="./output", show_all_metrics=False):
         results = {}
-        # 1. 扫描并算出所有注册的 Scorer 成绩
         for name, cls in SCORERS.items():
             try:
                 if "Align" in name:
@@ -94,44 +123,36 @@ class LatencyEvaluator:
                     )
                 else:
                     scorer = cls(computation_aware=computation_aware)
-                
+
                 score = scorer(self.instances)
                 key = f"{name}_CA" if computation_aware else name
                 results[key] = score
-            except Exception as e:
+            except Exception:
                 pass
-        
-        # 2. 智能化筛选最高级/最准确的一个版本展示给用户
+
         cleaned_results = {}
-        
-        # [A] 评估: 第一声开口延迟 (StartOffset)
+
         so_key = "StartOffset_CA" if computation_aware else "StartOffset"
         align_so_key = "StartOffset_SpeechAlign_CA" if computation_aware else "StartOffset_SpeechAlign"
-        
         if results.get(align_so_key) is not None and results[align_so_key] > 0:
             cleaned_results["First_Audio_Delay_(StartOffset_ms)"] = results[align_so_key]
         else:
             cleaned_results["First_Audio_Delay_(StartOffset_ms)"] = results.get(so_key, 0)
-            
-        # [B] 评估: 整句同传综合延迟 (标准版 ATD: 连同阅读/播放音频时间一起计算)
+
         atd_key = "ATD_CA" if computation_aware else "ATD"
         align_atd_key = "ATD_SpeechAlign_CA" if computation_aware else "ATD_SpeechAlign"
-        
         if results.get(align_atd_key) is not None and results[align_atd_key] > 0:
             cleaned_results["Overall_Translation_Delay_(ATD_ms)"] = results[align_atd_key]
         else:
             cleaned_results["Overall_Translation_Delay_(ATD_ms)"] = results.get(atd_key, 0)
 
-        # [C] 评估: 模型结单同传延迟 (纯净版 CustomATD: 剔除音频合成自身的物理时长，只看模型推断何时结束)
         catd_key = "CustomATD_CA" if computation_aware else "CustomATD"
         align_catd_key = "CustomATD_SpeechAlign_CA" if computation_aware else "CustomATD_SpeechAlign"
-        
         if results.get(align_catd_key) is not None and results[align_catd_key] > 0:
             cleaned_results["End_Action_Delay_(CustomATD_ms)"] = results[align_catd_key]
         else:
             cleaned_results["End_Action_Delay_(CustomATD_ms)"] = results.get(catd_key, 0)
 
-        # [D] 评估: 实时率指标 (RTF)
         if "RTF" in results:
             cleaned_results["Real_Time_Factor_(RTF)"] = results["RTF"]
         elif "RTF_CA" in results:
@@ -142,21 +163,19 @@ class LatencyEvaluator:
         elif "ModelGenerateRTF_CA" in results and results["ModelGenerateRTF_CA"] is not None:
             cleaned_results["Model_Generate_RTF"] = results["ModelGenerateRTF_CA"]
 
-        # ================= 修改开始 =================
-        # 如果用户显式开启了展示开关，则将原始杂乱数据装入 "detailed_all_metrics"
         if show_all_metrics:
             cleaned_results["detailed_all_metrics"] = results
-        # ================= 修改结束 =================
 
         return cleaned_results
 
+
 def load_agent_from_file(path, class_name):
-    """动态加载用户 Agent"""
     spec = importlib.util.spec_from_file_location("user_agent", path)
     mod = importlib.util.module_from_spec(spec)
     spec.loader.exec_module(mod)
     return getattr(mod, class_name)
 
+
 def main():
     parser = argparse.ArgumentParser(description="OpenSTBench Latency Evaluator")
     parser.add_argument("--source", required=True)
@@ -166,6 +185,7 @@ def main():
     parser.add_argument("--agent-script", required=True, help="Path to python file containing Agent class")
     parser.add_argument("--agent-class", required=True, help="Name of the Agent class")
     parser.add_argument("--segment-size", type=int, default=20)
+    parser.add_argument("--poll-interval-ms", type=float, default=10.0)
     parser.add_argument("--latency-unit", choices=["word", "char"], default="word")
     parser.add_argument("--disable-asr-fallback", action="store_true")
     parser.add_argument("--asr-model", default="medium")
@@ -179,20 +199,20 @@ def main():
     if args.slurm:
         submit_slurm(args, __file__)
 
-    # 1. 加载数据
-    with open(args.source) as f: src = [l.strip() for l in f if l.strip()]
-    ref = [None]*len(src)
+    with open(args.source) as f:
+        src = [l.strip() for l in f if l.strip()]
+    ref = [None] * len(src)
     if args.target:
-        with open(args.target) as f: ref = [l.strip() for l in f if l.strip()]
+        with open(args.target) as f:
+            ref = [l.strip() for l in f if l.strip()]
 
-    # 2. 加载用户 Agent
     AgentClass = load_agent_from_file(args.agent_script, args.agent_class)
     agent = AgentClass()
-    
-    # 3. 运行 Latency 评测
+
     evaluator = LatencyEvaluator(
         agent,
         args.segment_size,
+        poll_interval_ms=args.poll_interval_ms,
         latency_unit=args.latency_unit,
         asr_fallback_for_s2s_alignment=not args.disable_asr_fallback,
         asr_model=args.asr_model,
@@ -200,58 +220,47 @@ def main():
         alignment_dictionary_model=args.alignment_dictionary_model,
     )
     instances = evaluator.run(src, ref, args.task, args.output)
-    
-    # 4. 计算 Latency 指标
+
     print("\n--- Latency Metrics ---")
     scores = evaluator.compute_latency(False, args.output)
     if args.computation_aware:
         scores.update(evaluator.compute_latency(True, args.output))
-    
+
     for k, v in scores.items():
         print(f"{k:<25}: {v:.4f}")
 
-    # 5. [紧耦合] 运行 Quality 指标
     if args.quality and args.target:
         print("\n--- Quality Metrics (Integrated) ---")
-        
-        # 收集预测结果
-        # 注意: 对于 s2s, instances[i].prediction 应该是音频文件路径
         predictions = [ins.get_prediction_content() for ins in instances.values()]
-        
+
         if args.task == "s2t":
-            # [修改] 调用 TranslationEvaluator 算 BLEU
-            # 默认只开 BLEU 以保证速度
             qual_eval = TranslationEvaluator(
-                use_bleu=True, 
-                use_chrf=False, 
-                use_comet=False, 
-                use_whisper=False
+                use_bleu=True,
+                use_chrf=False,
+                use_comet=False,
+                use_whisper=False,
             )
-            # 使用新的 evaluate_all 接口
             q_res = qual_eval.evaluate_all(
-                target_text=predictions, 
-                reference=ref, 
-                source=src
+                target_text=predictions,
+                reference=ref,
+                source=src,
             )
             print(q_res)
-            
+
         elif args.task == "s2s":
-            # [修改] 调用 TranslationEvaluator 替代原 SpeechEvaluator
-            # 开启语音相关功能: use_wer=True, use_whisper=True
-            # predictions 是 wav 路径列表
             speech_eval = TranslationEvaluator(
-                use_wer=True, 
-                use_whisper=True, 
-                whisper_model="tiny", # 用 tiny 快速出 WER
-                use_utmos=False       # 可选开启
+                use_wer=True,
+                use_whisper=True,
+                whisper_model="tiny",
+                use_utmos=False,
             )
-            # 使用 evaluate_all 接口传入 target_speech
             s_res = speech_eval.evaluate_all(
-                target_speech=predictions, 
+                target_speech=predictions,
                 reference=ref,
-                source=src
+                source=src,
             )
             print(s_res)
 
+
 if __name__ == "__main__":
     main()

{openstbench-0.3.1 → openstbench-0.3.2}/src/openstbench/latency/instance.py

@@ -80,7 +80,8 @@ class SpeechToTextInstance(Instance):
     def len_sample_to_ms(self, length): return length * 1000 / self.sample_rate
 
     def send_source(self, segment_size=10):
-        if self.step == 0: self.start_time = time.time()
+        if self.step == 0 and self.start_time is None:
+            self.start_time = time.perf_counter()
         num_samples = math.ceil(segment_size / 1000 * self.sample_rate)
         
         if self.step < len(self.samples):
@@ -105,13 +106,13 @@ class SpeechToTextInstance(Instance):
         if getattr(seg, "config", None):
             self.add_model_inference_time(seg.config.get("model_inference_time"))
         
-        current_time = time.time()
+        current_time = time.perf_counter()
         content_list = seg.content.strip().split()
         self.prediction_list += content_list
         self.append_prediction_text(seg.content, source="native_transcript")
         
         curr_delay = self.len_sample_to_ms(self.step)
-        curr_elapsed = curr_delay + (current_time - self.start_time) * 1000
+        curr_elapsed = (current_time - self.start_time) * 1000
         
         self.delays += [curr_delay] * len(content_list)
         self.elapsed += [curr_elapsed] * len(content_list)
@@ -134,7 +135,8 @@ class SpeechToSpeechInstance(SpeechToTextInstance):
             self.finish_prediction = seg.finished
             return
         if not seg.content: return
-        if not self.start_time: self.start_time = time.time()
+        if not self.start_time:
+            self.start_time = time.perf_counter()
         if getattr(seg, "config", None):
             self.add_model_inference_time(seg.config.get("model_inference_time"))
             transcript = seg.config.get("transcript")
@@ -142,7 +144,7 @@ class SpeechToSpeechInstance(SpeechToTextInstance):
                 source = seg.config.get("transcript_source", "native_transcript")
                 self.append_prediction_text(transcript, source=source)
         
-        current_time = time.time()
+        current_time = time.perf_counter()
         duration_ms = len(seg.content) * 1000 / seg.sample_rate
         self.target_sample_rate = seg.sample_rate
         
@@ -150,7 +152,7 @@ class SpeechToSpeechInstance(SpeechToTextInstance):
         self.durations.append(duration_ms)
         
         curr_delay = self.len_sample_to_ms(self.step)
-        curr_elapsed = curr_delay + (current_time - self.start_time) * 1000
+        curr_elapsed = (current_time - self.start_time) * 1000
         
         self.delays.append(curr_delay)
         self.elapsed.append(curr_elapsed)

{openstbench-0.3.1 → openstbench-0.3.2}/src/openstbench/latency/metrics.py

@@ -86,6 +86,30 @@ class LatencyScorer:
     def subtract(self, arr1, arr2):
         return [x - y for x, y in zip(arr1, arr2)]
 
+    def get_incremental_compute_times(self, ins, count):
+        if not self.computation_aware:
+            return [0.0] * count
+
+        elapsed = list(getattr(ins, "elapsed", []) or [])
+        delays = list(getattr(ins, "delays", []) or [])
+        if not elapsed or not delays:
+            return [0.0] * count
+
+        limit = min(len(elapsed), len(delays), count)
+        if limit <= 0:
+            return [0.0] * count
+
+        compute_elapsed = [max(0.0, e - d) for e, d in zip(elapsed[:limit], delays[:limit])]
+        incremental = []
+        prev = 0.0
+        for value in compute_elapsed:
+            incremental.append(max(0.0, value - prev))
+            prev = value
+
+        if limit < count:
+            incremental.extend([0.0] * (count - limit))
+        return incremental
+
     def get_delays(self, ins):
         override = getattr(ins, "_alignment_delays", None)
         if override is not None:
@@ -226,12 +250,7 @@ class ATDScorer(LatencyScorer):
                         compute_times.append(0.0)
                 else:
                     base_emit_delays = list(getattr(ins, "delays", []) or [])
-                    text_compute_times = []
-                    if self.computation_aware and getattr(ins, "elapsed", None):
-                        compute_elapsed = self.subtract(ins.elapsed, ins.delays)
-                        text_compute_times = self.subtract(compute_elapsed, [0] + compute_elapsed[:-1])
-                    else:
-                        text_compute_times = [0.0] * len(delays)
+                    text_compute_times = self.get_incremental_compute_times(ins, len(delays))
 
                     for idx, delay in enumerate(delays):
                         emit_delay = base_emit_delays[idx] if idx < len(base_emit_delays) else delay
@@ -239,19 +258,18 @@ class ATDScorer(LatencyScorer):
                         target_delays.append(float(delay))
                         target_chunk_ids.append(source_chunk_id)
                         target_token_lens.append(0.0)
-                        compute_times.append(float(text_compute_times[idx]) if idx < len(text_compute_times) else 0.0)
+                        extra_compute = 0.0 if self.computation_aware else float(text_compute_times[idx])
+                        compute_times.append(extra_compute)
             else:
-                speech_compute_times = []
-                if self.computation_aware and getattr(ins, "elapsed", None):
-                    compute_elapsed = self.subtract(ins.elapsed, ins.delays)
-                    speech_compute_times = self.subtract(compute_elapsed, [0] + compute_elapsed[:-1])
-                else:
-                    speech_compute_times = [0.0] * len(delays)
+                speech_compute_times = self.get_incremental_compute_times(ins, len(delays))
 
                 for idx, duration_ms in enumerate(getattr(ins, "durations", []) or []):
                     source_chunk_id = self.source_chunk_id_from_delay(source_boundaries, ins.delays[idx])
                     token_lens = self.split_duration_into_tokens(duration_ms, token_len_ms=300.0)
-                    per_token_compute = float(speech_compute_times[idx]) / len(token_lens) if token_lens else 0.0
+                    if self.computation_aware:
+                        per_token_compute = 0.0
+                    else:
+                        per_token_compute = float(speech_compute_times[idx]) / len(token_lens) if token_lens else 0.0
                     for token_len in token_lens:
                         target_delays.append(float(delays[idx]))
                         target_chunk_ids.append(source_chunk_id)
@@ -328,31 +346,25 @@ class CustomATD(ATDScorer):
                         compute_times.append(0.0)
                 else:
                     base_emit_delays = list(getattr(ins, "delays", []) or [])
-                    text_compute_times = []
-                    if self.computation_aware and getattr(ins, "elapsed", None):
-                        compute_elapsed = self.subtract(ins.elapsed, ins.delays)
-                        text_compute_times = self.subtract(compute_elapsed, [0] + compute_elapsed[:-1])
-                    else:
-                        text_compute_times = [0.0] * len(delays)
+                    text_compute_times = self.get_incremental_compute_times(ins, len(delays))
 
                     for idx, delay in enumerate(delays):
                         emit_delay = base_emit_delays[idx] if idx < len(base_emit_delays) else delay
                         source_chunk_id = self.source_chunk_id_from_delay(source_boundaries, emit_delay)
                         target_delays.append(float(delay))
                         target_chunk_ids.append(source_chunk_id)
-                        compute_times.append(float(text_compute_times[idx]) if idx < len(text_compute_times) else 0.0)
+                        extra_compute = 0.0 if self.computation_aware else float(text_compute_times[idx])
+                        compute_times.append(extra_compute)
             else:
-                speech_compute_times = []
-                if self.computation_aware and getattr(ins, "elapsed", None):
-                    compute_elapsed = self.subtract(ins.elapsed, ins.delays)
-                    speech_compute_times = self.subtract(compute_elapsed, [0] + compute_elapsed[:-1])
-                else:
-                    speech_compute_times = [0.0] * len(delays)
+                speech_compute_times = self.get_incremental_compute_times(ins, len(delays))
 
                 for idx, duration_ms in enumerate(getattr(ins, "durations", []) or []):
                     source_chunk_id = self.source_chunk_id_from_delay(source_boundaries, ins.delays[idx])
                     token_lens = self.split_duration_into_tokens(duration_ms, token_len_ms=300.0)
-                    per_token_compute = float(speech_compute_times[idx]) / len(token_lens) if token_lens else 0.0
+                    if self.computation_aware:
+                        per_token_compute = 0.0
+                    else:
+                        per_token_compute = float(speech_compute_times[idx]) / len(token_lens) if token_lens else 0.0
                     for _ in token_lens:
                         target_delays.append(float(delays[idx]))
                         target_chunk_ids.append(source_chunk_id)