simulstream 0.1.0__py3-none-any.whl

simulstream/metrics/scorers/quality/sacrebleu.py

@@ -0,0 +1,59 @@
+# Copyright 2025 FBK
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+
+import argparse
+import logging
+import sys
+from typing import List
+
+from sacrebleu import BLEU
+
+from simulstream.metrics.scorers.quality import register_quality_scorer
+from simulstream.metrics.scorers.quality.mwersegmenter import MWERSegmenterBasedQualityScorer, \
+    ResegmentedQualityScoringSample
+
+try:
+    import sacrebleu
+except ImportError:
+    sys.exit("Please install comet first with `pip install sacrebleu`.")
+
+
+LOGGER = logging.getLogger('simulstream.metrics.scorers.latency.stream_laal')
+
+
+@register_quality_scorer("sacrebleu")
+class SacreBLEUScorer(MWERSegmenterBasedQualityScorer):
+    def __init__(self, args: argparse.Namespace):
+        super().__init__(args)
+        self.bleu = BLEU(tokenize=args.tokenizer)
+
+    def _do_score(self, samples: List[ResegmentedQualityScoringSample]) -> float:
+        hypotheses = []
+        references = []
+        for sample in samples:
+            hypotheses.extend(sample.hypothesis)
+            references.extend(sample.reference)
+        score = self.bleu.corpus_score(hypotheses, [references])
+        LOGGER.info(f"SacreBLEU signature: {self.bleu.get_signature()}")
+        LOGGER.info(f"SacreBLEU detailed score: {score}")
+        return score.score
+
+    @classmethod
+    def add_arguments(cls, parser: argparse.ArgumentParser) -> None:
+        parser.add_argument(
+            "--tokenizer", choices=sacrebleu.metrics.METRICS['BLEU'].TOKENIZERS,
+            default=sacrebleu.metrics.METRICS['BLEU'].TOKENIZER_DEFAULT)
+
+    def requires_source(self) -> bool:
+        return False

simulstream/metrics/stats.py

@@ -0,0 +1,184 @@
+# Copyright 2025 FBK
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+
+import argparse
+import json
+import logging
+from abc import abstractmethod
+
+import simulstream
+from simulstream.config import yaml_config
+from simulstream.metrics.readers import LogReader, text_items
+
+
+logging.basicConfig(
+    format='%(asctime)s | %(levelname)s | %(name)s | %(message)s',
+    datefmt='%Y-%m-%d %H:%M:%S',
+    level=logging.INFO,
+)
+LOGGER = logging.getLogger('simulstream.stats')
+
+
+class Stats:
+    """
+    Abstract base class for defining evaluation statistics.
+
+    Subclasses must implement:
+    - :meth:`name`: unique identifier of the statistic.
+    - :meth:`description`: a human-readable explanation.
+    - :meth:`compute`: logic to compute the metric from a :class:`LogReader`.
+    """
+    @abstractmethod
+    def name(self) -> str:
+        """The unique name of the statistic."""
+        ...
+
+    @abstractmethod
+    def description(self) -> str:
+        """The human-readable explanation of the statistic."""
+        ...
+
+    @abstractmethod
+    def compute(self, log_reader: LogReader) -> float:
+        """
+        Compute the value of the statistic.
+
+        Args:
+            log_reader (LogReader): Reader object encapsulating log data.
+
+        Returns:
+            float: The computed value of the statistic.
+        """
+        ...
+
+
+class NormalizedErasure(Stats):
+    """
+    Compute the **Normalized Erasure** metric.
+
+    This measures the amount of flickering in retranslation, as defined in
+    `Arivazhagan et al., "Re-translation versus Streaming for Simultaneous Translation"
+     IWSLT 2020 <https://aclanthology.org/2020.iwslt-1.27/>`_.
+
+    It is defined as the ratio:
+
+    .. math::
+
+        \\text{Normalized Erasure} =
+        \\frac{\\text{# Deleted Tokens}}{\\text{# Final Generated Tokens}}
+    """
+
+    def name(self) -> str:
+        return "normalized_erasure"
+
+    def description(self) -> str:
+        return "Normalized erasure, defined in https://aclanthology.org/2020.iwslt-1.27/, " \
+               "measures flickering in retranslation. It is defined as the ratio between the " \
+               "number of tokens that have been deleted and the number of final generated tokens."
+
+    def compute(self, log_reader: LogReader) -> float:
+        total_length = 0
+        for _, final_text in log_reader.final_outputs().items():
+            total_length += len(text_items(final_text, latency_unit=log_reader.latency_unit))
+        return log_reader.num_deleted_tokens() / total_length
+
+
+class RealTimeFactor(Stats):
+    """
+    Compute the **Real Time Factor**.
+
+    This measures how many seconds of computation are required on average
+    for each second of input audio.
+
+    Values greater than 1 indicate that the system is slower than real time
+    and cannot process input before the next audio chunk arrives.
+    """
+    def name(self) -> str:
+        return "real_time_factor"
+
+    def description(self) -> str:
+        return "The Real Time Factor measures the average computational cost, ie. time in " \
+               "seconds spent in computation, for each input audio second. Values higher than 1 " \
+               "mean that the system is not able to process the input in time before the next " \
+               "input arrives."
+
+    def compute(self, log_reader: LogReader) -> float:
+        total_audio_lengths = sum(
+            logs[-1]["total_audio_processed"] for _, logs in log_reader.outputs_by_audio.items())
+        total_computational_cost = sum(
+            sum(log["computation_time"] for log in logs)
+            for _, logs in log_reader.outputs_by_audio.items())
+        return total_computational_cost / total_audio_lengths
+
+
+def main(args: argparse.Namespace):
+    """
+    Main entry point for computing statistics.
+
+    Loads the evaluation configuration and log file, computes all defined
+    statistics, and prints them in JSON format.
+
+    Args:
+       args (argparse.Namespace): Parsed command-line arguments.
+    """
+    LOGGER.info(f"Loading evaluation configuration from {args.eval_config}")
+    eval_config = yaml_config(args.eval_config)
+    LOGGER.info(f"Loading evaluation log file from {args.log_file}")
+    log_reader = LogReader(eval_config, args.log_file, latency_unit=args.latency_unit)
+
+    LOGGER.info("Computing stats")
+    stats_classes = [NormalizedErasure(), RealTimeFactor()]
+    stats = {
+        stat.name(): {"description": stat.description(), "value": stat.compute(log_reader)}
+        for stat in stats_classes
+    }
+    print(f"Stats: {json.dumps(stats, indent=4)}")
+
+
+def cli_main():
+    """
+    Module for computing evaluation statistics from Simulstream logs.
+
+    This script provides a CLI interface to compute metrics that describe the behavior of
+    streaming systems. Metrics are computed from JSONL log files generated during evaluation and
+    include:
+
+    - **Normalized Erasure**: measures flickering in retranslation processors.
+    - **Computational Cost**: measures average computation time per second of audio.
+
+    The output is printed on standard output in JSON format.
+
+    Typical usage from the command line:
+
+        $ python -m simulstream.metrics.stats --eval-config config/speech_processor.yaml \\
+            --log-file metrics.jsonl
+    """
+    LOGGER.info(f"Simulstream version: {simulstream.__version__}")
+    parser = argparse.ArgumentParser("stats")
+    parser.add_argument(
+        "--eval-config", type=str, required=True,
+        help="Path to the yaml config file containing information about the tokenizer to be used.")
+    parser.add_argument(
+        "--log-file", type=str, required=True,
+        help="Path to the log file with the metrics to be used for the evaluation.")
+    parser.add_argument(
+        "--latency-unit", choices=["word", "char"], default="word",
+        help="Whether to computed stats based on words or characters. Default: word.")
+
+    args = parser.parse_args()
+    main(args)
+
+
+if __name__ == "__main__":
+    cli_main()

simulstream/server/http_server.py

@@ -0,0 +1,95 @@
+# Copyright 2025 FBK
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+
+import argparse
+import logging
+from functools import partial
+from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
+
+import simulstream
+
+
+logging.basicConfig(
+    format='%(asctime)s | %(levelname)s | %(name)s | %(message)s',
+    datefmt='%Y-%m-%d %H:%M:%S',
+    level=logging.INFO,
+)
+LOGGER = logging.getLogger('simulstream.http_server')
+
+
+class CustomHandler(SimpleHTTPRequestHandler):
+    def __init__(self, *args, config=None, **kwargs):
+        self.config = config
+        super().__init__(*args, **kwargs)
+
+    def do_GET(self):
+        if self.path == "/config.yaml":
+            # Load template
+            with open(self.config) as f:
+                config = f.read()
+
+            # Send response
+            self.send_response(200)
+            self.send_header("Content-type", "text/yaml; charset=utf-8")
+            self.end_headers()
+            self.wfile.write(config.encode("utf-8"))
+        else:
+            super().do_GET()
+
+
+def cli_main():
+    """
+    Simulstream HTTP server command-line interface (CLI) entry point.
+
+    This function parses command-line arguments and starts the asynchronous :func:`main` routine.
+
+    Example usage::
+
+        $ python http_server.py --config config/server.yaml --directory webdemo
+
+    Command-line arguments:
+
+    - ``bind`` (str): IP/address on which to serve [default: 127.0.0.1].
+    - ``port`` (int): Port on which to serve [default: 8000].
+    - ``--config`` (str): Path to the server configuration file.
+    - ``--directory`` (str): Path to the server configuration file.
+
+    .. note::
+        The server currently does not support secure connection through HTTPS
+    """
+    LOGGER.info(f"HTTP server version: {simulstream.__version__}")
+    parser = argparse.ArgumentParser(description="Simulstream http.server")
+    parser.add_argument(
+        "--bind", "-b", default="127.0.0.1",
+        help="Specify alternate bind address [default: 127.0.0.1]")
+    parser.add_argument(
+        "--port", "-p", type=int, default=8000,
+        help="Specify alternate port [default: 8000]")
+    parser.add_argument(
+        "--config", "-c", required=True,
+        help="Path to configuration file (YAML)")
+    parser.add_argument(
+        "--directory", "-d", default="./webdemo",
+        help="Path to the directory containing the HTML web demo [default: ./webdemo]")
+    args = parser.parse_args()
+
+    custom_handler = partial(CustomHandler, config=args.config, directory=args.directory)
+    httpd = ThreadingHTTPServer((args.bind, args.port), custom_handler)
+    LOGGER.info(f"Serving directory {args.directory}")
+    LOGGER.info(f"Serving on http://{args.bind}:{args.port}")
+    httpd.serve_forever()
+
+
+if __name__ == "__main__":
+    cli_main()

simulstream/server/message_processor.py

@@ -0,0 +1,156 @@
+# Copyright 2025 FBK
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+
+import json
+import logging
+import time
+from typing import Optional
+
+import librosa
+import numpy as np
+
+from simulstream.metrics.logger import METRICS_LOGGER
+from simulstream.server.speech_processors import SpeechProcessor, SAMPLE_RATE
+from simulstream.server.speech_processors.incremental_output import merge_incremental_outputs, \
+    IncrementalOutput
+
+
+LOGGER = logging.getLogger('simulstream.message_processor')
+
+
+class MessageProcessor:
+    """
+    This class is responsible for processing the messages incoming from a client, which include
+    control messages (e.g., configurations about languages to use, or signal that the stream is
+    over).
+    """
+    def __init__(self, client_id: int, speech_processor: SpeechProcessor):
+        self.client_buffer = b''
+        self.processed_audio_seconds = 0
+        self.sample_rate = SAMPLE_RATE
+        self.client_id = client_id
+        self.speech_processor = speech_processor
+
+    def process_speech(self, speech_data: bytes) -> Optional[IncrementalOutput]:
+        """
+        Process an audio chunk and return incremental transcription/translation.
+        Namely, it receives and buffers raw audio chunks (``bytes``) and processes audio
+        audio incrementally with the configured
+        :class:`~simulstream.server.speech_processors.SpeechProcessor`.
+
+        Args:
+            speech_data (bytes): Raw PCM audio bytes (16-bit little endian).
+
+        Returns:
+            IncrementalOutput: incremental processing results, if any. None otherwise.
+        """
+        self.client_buffer += speech_data
+        # we have SAMPLE_RATE * 2 bytes (int16) samples every second
+        buffer_len_seconds = len(self.client_buffer) / 2 / self.sample_rate
+        if buffer_len_seconds >= self.speech_processor.speech_chunk_size:
+            self.processed_audio_seconds += buffer_len_seconds
+            start_time = time.time()
+            incremental_output = self._run_speech_processor()
+            end_time = time.time()
+            METRICS_LOGGER.info(json.dumps({
+                "id": self.client_id,
+                "total_audio_processed": self.processed_audio_seconds,
+                "computation_time": end_time - start_time,
+                "generated_tokens": incremental_output.new_tokens,
+                "deleted_tokens": incremental_output.deleted_tokens,
+            }))
+            return incremental_output
+        else:
+            return None
+
+    def _run_speech_processor(self) -> IncrementalOutput:
+        """
+        This function converts raw ``int16`` PCM audio to normalized ``float32``,
+        resamples it if necessary to :data:`~simulstream.server.speech_processors.SAMPLE_RATE`,
+        and forwards it to the given class:`~simulstream.server.speech_processors.SpeechProcessor`.
+        Processing statistics are logged using the metrics logger.
+        """
+        int16_waveform = np.frombuffer(self.client_buffer, dtype=np.int16)
+        float32_waveform = int16_waveform.astype(np.float32) / 2 ** 15
+        if self.sample_rate != SAMPLE_RATE:
+            float32_waveform = librosa.resample(
+                float32_waveform, orig_sr=self.sample_rate, target_sr=SAMPLE_RATE)
+        incremental_output = self.speech_processor.process_chunk(float32_waveform)
+        self.client_buffer = b''
+        return incremental_output
+
+    def process_metadata(self, metadata: dict):
+        """
+        Takes a dictionary of metadata regarding the incoming speech and desired output, and
+        interacts with the configured speech_processor to set it up accordingly.
+
+        Args:
+            metadata (dict): Dictionary of metadata regarding the incoming speech.
+        """
+        if 'sample_rate' in metadata:
+            self.sample_rate = int(metadata['sample_rate'])
+        if 'target_lang' in metadata:
+            self.speech_processor.set_target_language(metadata["target_lang"])
+            LOGGER.debug(
+                f"Client {self.client_id} target language set to: "
+                f"{metadata['target_lang']}")
+        if 'source_lang' in metadata:
+            self.speech_processor.set_source_language(metadata["source_lang"])
+            LOGGER.debug(
+                f"Client {self.client_id} source language set to: {metadata['source_lang']}")
+        if 'metrics_metadata' in metadata:
+            METRICS_LOGGER.info(json.dumps({
+                "id": self.client_id,
+                "metadata": metadata["metrics_metadata"]
+            }))
+            LOGGER.debug(
+                f"Logged client {self.client_id} metrics metadata: {metadata['metrics_metadata']}")
+
+    def end_of_stream(self) -> IncrementalOutput:
+        """
+        Performs the last operations to conclude the processing of the stream of audio by the
+        speech processor and cleans up everything to be ready for the next stream.
+
+        Returns:
+            IncrementalOutput: last output at the end of the stream.
+        """
+        outputs = []
+        start_time = time.time()
+        if self.client_buffer:
+            # process remaining audio after last chunk
+            self.processed_audio_seconds += len(self.client_buffer) / 2 / self.sample_rate
+            outputs.append(self._run_speech_processor())
+
+        outputs.append(self.speech_processor.end_of_stream())
+        incremental_output = merge_incremental_outputs(
+            outputs, self.speech_processor.tokens_to_string)
+        end_time = time.time()
+        METRICS_LOGGER.info(json.dumps({
+            "id": self.client_id,
+            "total_audio_processed": self.processed_audio_seconds,
+            "computation_time": end_time - start_time,
+            "generated_tokens": incremental_output.new_tokens,
+            "deleted_tokens": incremental_output.deleted_tokens,
+        }))
+        self.clear()
+        return incremental_output
+
+    def clear(self):
+        """
+        Clear the internal states to be ready for a new input stream.
+        """
+        self.speech_processor.clear()
+        self.client_buffer = b''
+        self.processed_audio_seconds = 0
+        self.sample_rate = SAMPLE_RATE

simulstream/server/speech_processors/__init__.py

@@ -0,0 +1,173 @@
+# Copyright 2025 FBK
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+
+import importlib
+from abc import ABC, abstractmethod
+from types import SimpleNamespace
+from typing import List, Any
+
+import numpy as np
+
+from simulstream.server.speech_processors.incremental_output import IncrementalOutput
+
+
+CHANNELS = 1
+SAMPLE_WIDTH = 2
+SAMPLE_RATE = 16_000
+
+
+class SpeechProcessor(ABC):
+    """
+    Abstract base class for speech processors.
+
+    Subclasses must implement methods to load models, process audio chunks,
+    set source/target languages, and clear internal states.
+    """
+
+    def __init__(self, config: SimpleNamespace):
+        """
+        Initialize the speech processor with a given configuration.
+
+        Args:
+            config (SimpleNamespace): Configuration loaded from a YAML file.
+        """
+        self.config = config
+
+    @property
+    def speech_chunk_size(self) -> float:
+        """
+        Return the size of the speech chunks to be processed (in seconds).
+        """
+        return self.config.speech_chunk_size
+
+    @classmethod
+    @abstractmethod
+    def load_model(cls, config: SimpleNamespace):
+        """
+        Load and initialize the underlying speech model.
+
+        Args:
+            config (SimpleNamespace): Configuration of the speech processor.
+        """
+        ...
+
+    @abstractmethod
+    def process_chunk(self, waveform: np.float32) -> IncrementalOutput:
+        """
+        Process a chunk of waveform and produce incremental output.
+
+        Args:
+            waveform (np.float32): A 1D NumPy array of the audio chunk. The array is PCM audio
+                normalized to the range ``[-1.0, 1.0]`` sampled at
+                :attr:`simulstream.server.speech_processors.SAMPLE_RATE`.
+
+        Returns:
+            IncrementalOutput: The incremental output (new and deleted tokens/strings).
+        """
+        ...
+
+    @abstractmethod
+    def set_source_language(self, language: str) -> None:
+        """
+        Set the source language for the speech processor.
+
+        Args:
+            language (str): Language code (e.g., ``"en"``, ``"it"``).
+        """
+        ...
+
+    @abstractmethod
+    def set_target_language(self, language: str) -> None:
+        """
+        Set the target language for the speech processor (for translation).
+
+        Args:
+            language (str): Language code (e.g., ``"en"``, ``"it"``).
+        """
+        ...
+
+    @abstractmethod
+    def end_of_stream(self) -> IncrementalOutput:
+        """
+        This method is called at the end of audio chunk processing. It can be used to emit
+        hypotheses at the end of the speech to conclude the output.
+
+        Returns:
+            IncrementalOutput: The incremental output (new and deleted tokens/strings).
+        """
+        ...
+
+    @abstractmethod
+    def tokens_to_string(self, tokens: List[str]) -> str:
+        """
+        Converts token sequences into human-readable strings.
+
+        Returns:
+            str: The textual representation of the tokens.
+        """
+        ...
+
+    @abstractmethod
+    def clear(self) -> None:
+        """
+        Clear internal states, such as history of cached audio and/or tokens,
+        in preparation for a new stream or conversation.
+        """
+        ...
+
+
+def build_speech_processor(speech_processor_config: SimpleNamespace) -> SpeechProcessor:
+    """
+    Instantiate a SpeechProcessor subclass based on configuration.
+
+    The configuration should specify the fully-qualified class name in the
+    ``type`` field (e.g. ``"simulstream.server.speech_processors.MyProcessor"``).
+
+    Args:
+        speech_processor_config (SimpleNamespace): Configuration for the speech processor.
+
+    Returns:
+        SpeechProcessor: An instance of the configured speech processor.
+
+    Raises:
+        AssertionError: If the specified class is not a subclass of SpeechProcessor.
+    """
+    cls = speech_processor_class_load(speech_processor_config.type)
+    cls.load_model(speech_processor_config)
+    return cls(speech_processor_config)
+
+
+def speech_processor_class_load(speech_processor_class_string: str) -> type[SpeechProcessor]:
+    """
+    Import the speech processor class from its string definition.
+
+    Args:
+        speech_processor_class_string (str): Full name of the speech processor class to load.
+
+    Returns:
+        SpeechProcessorClass: A class object for the speech processor class.
+
+    Raises:
+        AssertionError: If the specified class is not a subclass of SpeechProcessor.
+    """
+    cls = class_load(speech_processor_class_string)
+    assert issubclass(cls, SpeechProcessor), \
+        f"{speech_processor_class_string} must be a subclass of SpeechProcessor"
+    return cls
+
+
+def class_load(class_string: str) -> type[Any]:
+    module_path, class_name = class_string.rsplit('.', 1)
+    module = importlib.import_module(module_path)
+    return getattr(module, class_name)