simulstream 0.1.0__py3-none-any.whl

simulstream/metrics/scorers/latency/__init__.py

@@ -0,0 +1,115 @@
+# Copyright 2025 FBK
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+
+import argparse
+import importlib
+import pkgutil
+from abc import abstractmethod
+from dataclasses import dataclass
+from typing import List, Optional
+
+from simulstream.metrics.readers import OutputWithDelays, ReferenceSentenceDefinition
+
+
+LATENCY_SCORER_REGISTRY = {}
+
+
+def register_latency_scorer(name):
+    """
+    Decorator for registering a latency scorer class.
+
+    Args:
+        name (str): The unique identifier for the scorer.
+
+    Raises:
+        TypeError: If the decorated class is not a subclass of
+            :class:`LatencyScorer`.
+
+    Example:
+        >>> @register_latency_scorer("stream_laal")
+        ... class StreamLAALScorer(LatencyScorer):
+        ...     ...
+    """
+    def register(cls):
+        if not issubclass(cls, LatencyScorer):
+            raise TypeError(f"Cannot register {cls.__name__}: must be a subclass of LatencyScorer")
+        LATENCY_SCORER_REGISTRY[name] = cls
+        return cls
+
+    return register
+
+
+@dataclass
+class LatencyScoringSample:
+    """
+    Data structure representing a single evaluation sample.
+
+    Attributes:
+        audio_name (str): The identifier of the audio file.
+        hypothesis (str): The system-generated hypothesis text.
+        reference (Optional[List[ReferenceSentenceDefinition]]): One or more reference sentences,
+            including the text, start time and duration, or ``None`` if not required.
+    """
+    audio_name: str
+    hypothesis: OutputWithDelays
+    reference: Optional[List[ReferenceSentenceDefinition]] = None
+
+
+@dataclass
+class LatencyScores:
+    """
+    Data structure representing a latency score.
+
+    Attributes:
+        ideal_latency (float): The latency score in ideal conditions, which do not include
+            computational costs.
+        computational_aware_latency (Optional[float]): The latency score in computational aware
+            conditions, which include computational costs.
+    """
+    ideal_latency: float
+    computational_aware_latency: Optional[float] = None
+
+
+class LatencyScorer:
+    """
+    Abstract base class for all latency scorers.
+
+    A latency scorer evaluates system hypotheses against references and returns a
+    :class:`LatencyScores` object that represents the latency scores.
+
+    Subclasses must implement the abstract methods defined here and should be registered via
+    :func:`register_latency_scorer`.
+
+    Args:
+        args (argparse.Namespace): Parsed command-line arguments.
+    """
+    def __init__(self, args: argparse.Namespace):
+        self.args = args
+
+    @abstractmethod
+    def score(self, samples: List[LatencyScoringSample]) -> LatencyScores:
+        ...
+
+    @classmethod
+    @abstractmethod
+    def add_arguments(cls, parser: argparse.ArgumentParser) -> None:
+        ...
+
+    @abstractmethod
+    def requires_reference(self) -> bool:
+        ...
+
+
+for loader, name, is_pkg in pkgutil.walk_packages(__path__, __name__ + "."):
+    importlib.import_module(name)

simulstream/metrics/scorers/latency/mwersegmenter.py

@@ -0,0 +1,136 @@
+# Copyright 2025 FBK
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+
+from abc import abstractmethod
+from dataclasses import dataclass
+from typing import List
+
+from mweralign import mweralign
+
+from simulstream.metrics.readers import ReferenceSentenceDefinition, OutputWithDelays, text_items
+from simulstream.metrics.scorers.latency import LatencyScorer, LatencyScoringSample, LatencyScores
+
+
+@dataclass
+class ResegmentedLatencyScoringSample:
+    """
+    A sample containing realigned hypotheses and references.
+
+    Attributes:
+        audio_name (str): The identifier of the audio file.
+        hypothesis (List[str]): Hypothesis lines after realignment.
+        reference (List[str]): Reference lines aligned to the hypothesis.
+    """
+    audio_name: str
+    hypothesis: List[OutputWithDelays]
+    reference: List[ReferenceSentenceDefinition]
+
+
+class MWERSegmenterBasedLatencyScorer(LatencyScorer):
+    """
+    Abstract base class for scorers that require aligned system outputs and references through
+    MWER Segmenter alignment.
+
+    This class wraps a latency scorer and applies the MWER Segmenter alignment by `"Effects of
+    automatic alignment on speech translation metrics"
+    <https://aclanthology.org/2025.iwslt-1.7/>`_ to hypotheses before scoring.
+
+    Subclasses must implement :meth:`_do_score`, which operates on
+    :class:`ResegmentedLatencyScoringSample` instances where hypotheses and references are aligned.
+
+    Example:
+        >>> class CustomLatencyScorer(MWERSegmenterBasedLatencyScorer):
+        ...     def _do_score(self, samples):
+        ...         # Compute a custom latency score
+        ...         return LatencyScores(...)
+    """
+    def __init__(self, args):
+        super().__init__(args)
+        self.latency_unit = args.latency_unit
+
+    def requires_reference(self) -> bool:
+        return True
+
+    @abstractmethod
+    def _do_score(self, samples: List[ResegmentedLatencyScoringSample]) -> LatencyScores:
+        """
+        Compute latency scores on resegmented samples.
+
+        Subclasses must override this method.
+
+        Args:
+            samples (List[ResegmentedLatencyScoringSample]): Aligned
+                hypothesis–reference pairs with delay information.
+
+        Returns:
+            LatencyScores: The computed latency metrics.
+        """
+        ...
+
+    def _split_delays_by_segmented_text(
+            self, delays: List[float], segmented_text: List[str]) -> List[List[float]]:
+        """
+        Assign delay values to the corresponding segmented hypotheses.
+
+        Args:
+            delays (List[float]): Delay values (per token or per char).
+            segmented_text (List[str]): Segmented hypothesis strings.
+
+        Returns:
+            List[List[float]]: Delays split per segment.
+        """
+        segmented_delays = []
+        index = 0
+
+        for segment in segmented_text:
+            segment_len = len(text_items(segment, self.latency_unit))
+            segmented_delays.append(delays[index:index + segment_len])
+            index += segment_len
+        assert len(delays) == index, \
+            f"Index {index} should have reached end of delays ({len(delays)})"
+        return segmented_delays
+
+    def score(self, samples: List[LatencyScoringSample]) -> LatencyScores:
+        resegmented_samples = []
+        for sample in samples:
+            assert sample.reference is not None, "Cannot realign hypothesis to missing reference"
+
+            resegmented_hypos = mweralign.align_texts(
+                "\n".join([sentence_def.content for sentence_def in sample.reference]),
+                sample.hypothesis.final_text).split("\n")
+
+            assert len(resegmented_hypos) == len(sample.reference), \
+                f"Reference ({sample.audio_name}) has mismatched number of target " \
+                f"({len(sample.reference)}) and resegmented lines ({len(resegmented_hypos)})"
+
+            ideal_delays_splits = self._split_delays_by_segmented_text(
+                sample.hypothesis.ideal_delays,
+                resegmented_hypos)
+            computational_aware_delays_splits = self._split_delays_by_segmented_text(
+                sample.hypothesis.computational_aware_delays,
+                resegmented_hypos)
+            assert len(ideal_delays_splits) == len(computational_aware_delays_splits)
+
+            resegmented_hypos_with_delays = []
+            for text, ideal_delay, computational_aware_delay in zip(
+                    resegmented_hypos, ideal_delays_splits, computational_aware_delays_splits):
+                resegmented_hypos_with_delays.append(
+                    OutputWithDelays(text, ideal_delay, computational_aware_delay))
+
+            resegmented_samples.append(ResegmentedLatencyScoringSample(
+                sample.audio_name,
+                resegmented_hypos_with_delays,
+                sample.reference,
+            ))
+        return self._do_score(resegmented_samples)

simulstream/metrics/scorers/latency/stream_laal.py

@@ -0,0 +1,119 @@
+# Copyright 2025 FBK
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+
+import argparse
+import logging
+import statistics
+from typing import List
+
+from simulstream.metrics.readers import text_items
+from simulstream.metrics.scorers.latency import register_latency_scorer, LatencyScores
+from simulstream.metrics.scorers.latency.mwersegmenter import MWERSegmenterBasedLatencyScorer, \
+    ResegmentedLatencyScoringSample
+
+
+LOGGER = logging.getLogger('simulstream.metrics.scorers.latency.stream_laal')
+
+
+@register_latency_scorer("stream_laal")
+class StreamLaal(MWERSegmenterBasedLatencyScorer):
+    """
+    Computes StreamLAAL version 2.0, as proposed in
+    `StreamAtt: Direct Streaming Speech-to-Text Translation with Attention-based
+    Audio History Selection <https://aclanthology.org/2024.acl-long.202.pdf>`_.
+
+
+    Then main difference with version 1 is the different segmentation of
+    the text (uses mwerSegmenter python package instead of Matusov's executable).
+    """
+
+    @staticmethod
+    def _sentence_level_laal(
+            delays: List[float], source_length: float, target_length: int) -> float:
+        """
+        Function to compute Length Adaptive Average Lagging (LAAL) on one sentence as proposed in
+        `CUNI-KIT System for Simultaneous Speech Translation Task at IWSLT 2022
+        <https://arxiv.org/abs/2204.06028>`_ and
+        `Length-Adaptive Average Lagging for Simultaneous Speech Translation
+        <https://arxiv.org/abs/2206.05807>`_.
+        It is the original Average Lagging as proposed in
+        `Controllable Latency using Prefix-to-Prefix Framework
+        <https://arxiv.org/abs/1810.08398>`_
+        but is robust to the length difference between the hypothesis and reference.
+
+        The implementation is derived by that available in SimulEval (see `latency_scorer.py` in
+        `https://github.com/facebookresearch/SimulEval/).
+
+        Returns:
+            float: the latency score on one sentence.
+        """
+        if delays[0] > source_length:
+            return delays[0]
+
+        LAAL = 0
+        gamma = max(len(delays), target_length) / source_length
+        tau = 0
+        for t_minus_1, d in enumerate(delays):
+            LAAL += d - t_minus_1 / gamma
+            tau = t_minus_1 + 1
+
+            if d >= source_length:
+                break
+        LAAL /= tau
+        return LAAL
+
+    def _do_score(self, samples: List[ResegmentedLatencyScoringSample]) -> LatencyScores:
+        sentence_level_ideal_scores = []
+        sentence_level_ca_scores = []
+        skipped_sentences = 0
+        for sample in samples:
+            for sentence_output, sentence_reference in zip(sample.hypothesis, sample.reference):
+                # offset delays with respect to reference start of the utterance
+                delays_from_sentence_start = [
+                    delay - sentence_reference.start_time
+                    for delay in sentence_output.ideal_delays]
+                ca_delays_from_sentence_start = [
+                    delay - sentence_reference.start_time
+                    for delay in sentence_output.computational_aware_delays]
+                assert len(delays_from_sentence_start) == len(ca_delays_from_sentence_start)
+
+                target_length = len(text_items(sentence_reference.content, self.latency_unit))
+
+                if len(delays_from_sentence_start) > 0:
+                    sentence_level_ideal_scores.append(
+                        self._sentence_level_laal(
+                            delays_from_sentence_start,
+                            sentence_reference.duration,
+                            target_length)
+                    )
+                    sentence_level_ca_scores.append(
+                        self._sentence_level_laal(
+                            ca_delays_from_sentence_start,
+                            sentence_reference.duration,
+                            target_length)
+                    )
+                else:
+                    skipped_sentences += 1
+
+        if skipped_sentences > 0:
+            LOGGER.warning(
+                f"{skipped_sentences} sentences have been skipped in LAAL computation as they "
+                "were empty")
+        return LatencyScores(
+            statistics.mean(sentence_level_ideal_scores),
+            statistics.mean(sentence_level_ca_scores))
+
+    @classmethod
+    def add_arguments(cls, parser: argparse.ArgumentParser) -> None:
+        pass

simulstream/metrics/scorers/quality/__init__.py

@@ -0,0 +1,132 @@
+# Copyright 2025 FBK
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+
+import argparse
+import importlib
+import pkgutil
+from abc import abstractmethod
+from dataclasses import dataclass
+from typing import List, Optional
+
+
+QUALITY_SCORER_REGISTRY = {}
+
+
+def register_quality_scorer(name):
+    """
+    Decorator for registering a quality scorer class.
+
+    Args:
+        name (str): The unique identifier for the scorer.
+
+    Raises:
+        TypeError: If the decorated class is not a subclass of
+            :class:`QualityScorer`.
+
+    Example:
+        >>> @register_quality_scorer("bleu")
+        ... class BLEUScorer(QualityScorer):
+        ...     ...
+    """
+    def register(cls):
+        if not issubclass(cls, QualityScorer):
+            raise TypeError(f"Cannot register {cls.__name__}: must be a subclass of QualityScorer")
+        QUALITY_SCORER_REGISTRY[name] = cls
+        return cls
+
+    return register
+
+
+@dataclass
+class QualityScoringSample:
+    """
+    Data structure representing a single evaluation sample.
+
+    Attributes:
+        audio_name (str): The identifier of the audio file.
+        hypothesis (str): The system-generated hypothesis text.
+        reference (Optional[List[str]]): One or more reference translations, or ``None`` if not
+            required.
+        source (Optional[List[str]]): The source transcription or text, or ``None`` if not
+            required.
+    """
+    audio_name: str
+    hypothesis: str
+    reference: Optional[List[str]] = None
+    source: Optional[List[str]] = None
+
+
+class QualityScorer:
+    """
+    Abstract base class for all quality scorers.
+
+    A quality scorer evaluates system hypotheses against references and/or source sentences
+    and returns a numerical score.
+
+    Subclasses must implement the abstract methods defined here and should be registered via
+    :func:`register_quality_scorer`.
+
+    Args:
+        args (argparse.Namespace): Parsed command-line arguments.
+    """
+    def __init__(self, args: argparse.Namespace):
+        self.args = args
+
+    @abstractmethod
+    def score(self, samples: List[QualityScoringSample]) -> float:
+        """
+        Compute a quality score over a list of samples.
+
+        Args:
+           samples (List[QualityScoringSample]): Samples to be evaluated.
+
+        Returns:
+           float: The computed quality score.
+        """
+        ...
+
+    @classmethod
+    @abstractmethod
+    def add_arguments(cls, parser: argparse.ArgumentParser) -> None:
+        """
+        Add scorer-specific arguments to the CLI parser.
+
+        Args:
+            parser (argparse.ArgumentParser): The parser to extend.
+        """
+        ...
+
+    @abstractmethod
+    def requires_source(self) -> bool:
+        """
+        Indicate whether this scorer requires the source text.
+
+        Returns:
+            bool: True if source sentences are required, False otherwise.
+        """
+        ...
+
+    @abstractmethod
+    def requires_reference(self) -> bool:
+        """
+        Indicate whether this scorer requires reference translations.
+
+        Returns:
+            bool: True if references are required, False otherwise.
+        """
+        ...
+
+
+for loader, name, is_pkg in pkgutil.walk_packages(__path__, __name__ + "."):
+    importlib.import_module(name)

simulstream/metrics/scorers/quality/comet.py

@@ -0,0 +1,57 @@
+# Copyright 2025 FBK
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+
+import argparse
+import sys
+from typing import List
+
+from simulstream.metrics.scorers.quality import register_quality_scorer
+from simulstream.metrics.scorers.quality.mwersegmenter import MWERSegmenterBasedQualityScorer, \
+    ResegmentedQualityScoringSample
+
+try:
+    from comet import download_model, load_from_checkpoint
+except ImportError:
+    sys.exit("Please install comet first with `pip install unbabel-comet`.")
+
+
+@register_quality_scorer("comet")
+class CometScorer(MWERSegmenterBasedQualityScorer):
+    def __init__(self, args: argparse.Namespace):
+        super().__init__(args)
+        self.batch_size = args.batch_size
+        model_path = download_model(args.model)
+        self.model = load_from_checkpoint(model_path)
+        self.model.eval()
+
+    def _do_score(self, samples: List[ResegmentedQualityScoringSample]) -> float:
+        comet_data = []
+        for sample in samples:
+            for hyp, ref, src in zip(sample.hypothesis, sample.reference, sample.source):
+                comet_data.append({
+                    "src": src.strip(),
+                    "mt": hyp.strip(),
+                    "ref": ref.strip()
+                })
+
+        metric = self.model.predict(comet_data, batch_size=self.batch_size)
+        return metric.system_score
+
+    @classmethod
+    def add_arguments(cls, parser: argparse.ArgumentParser) -> None:
+        parser.add_argument("--model", type=str, default="Unbabel/wmt22-comet-da")
+        parser.add_argument("--batch-size", type=int, default=16)
+
+    def requires_source(self) -> bool:
+        return True

simulstream/metrics/scorers/quality/mwersegmenter.py

@@ -0,0 +1,93 @@
+# Copyright 2025 FBK
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+
+from abc import abstractmethod
+from dataclasses import dataclass
+from typing import List, Optional
+
+from mweralign import mweralign
+
+from simulstream.metrics.scorers.quality import QualityScorer, QualityScoringSample
+
+
+@dataclass
+class ResegmentedQualityScoringSample:
+    """
+    A sample containing realigned hypotheses and references.
+
+    Attributes:
+        audio_name (str): The identifier of the audio file.
+        hypothesis (List[str]): Hypothesis lines after realignment.
+        reference (List[str]): Reference lines aligned to the hypothesis.
+        source (Optional[List[str]]): Source text (if available).
+    """
+    audio_name: str
+    hypothesis: List[str]
+    reference: List[str]
+    source: Optional[List[str]] = None
+
+
+class MWERSegmenterBasedQualityScorer(QualityScorer):
+    """
+    Abstract base class for scorers that require aligned system outputs and references through
+    MWER Segmenter alignment.
+
+    This class wraps a quality scorer and applies the MWER Segmenter alignment by `"Effects of
+    automatic alignment on speech translation metrics"
+    <https://aclanthology.org/2025.iwslt-1.7/>`_ to hypotheses before scoring.
+
+    Subclasses must implement :meth:`_do_score`, which receives
+    :class:`ResegmentedQualityScoringSample` instances, where output and references are aligned.
+
+    Example:
+        >>> class CustomQualityScorer(MWERSegmenterBasedQualityScorer):
+        ...     def _do_score(self, samples):
+        ...         # Compute a custom quality score
+        ...         return ...
+    """
+    def requires_reference(self) -> bool:
+        return True
+
+    @abstractmethod
+    def _do_score(self, samples: List[ResegmentedQualityScoringSample]) -> float:
+        """
+        Compute the final score on resegmented samples.
+
+        This method must be implemented by subclasses.
+
+        Args:
+            samples (List[ResegmentedQualityScoringSample]): The aligned
+                hypothesis–reference pairs, plus optional sources.
+
+        Returns:
+            float: The computed score.
+        """
+        ...
+
+    def score(self, samples: List[QualityScoringSample]) -> float:
+        resegmented_samples = []
+        for sample in samples:
+            assert sample.reference is not None, "Cannot realign hypothesis to missing reference"
+            resegmented_hypos = mweralign.align_texts(
+                "\n".join(sample.reference), sample.hypothesis).split("\n")
+            assert len(sample.reference) == len(resegmented_hypos), \
+                f"Reference ({sample.audio_name}) has mismatched number of target " \
+                f"({len(sample.reference)}) and resegmented lines ({len(resegmented_hypos)})"
+            resegmented_samples.append(ResegmentedQualityScoringSample(
+                sample.audio_name,
+                resegmented_hypos,
+                sample.reference,
+                sample.source
+            ))
+        return self._do_score(resegmented_samples)