simulstream 0.3.0__tar.gz → 1.0.0__tar.gz

{simulstream-0.3.0/simulstream.egg-info → simulstream-1.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: simulstream
-Version: 0.3.0
+Version: 1.0.0
 Summary: A server to run simultaneous/streaming experiments and demo
 Author-email: Marco Gaido <mgaido@fbk.eu>, FBK HLT-MT <mt@fbk.eu>
 License:                                  Apache License
@@ -215,6 +215,7 @@ Requires-Dist: pyyaml>6.0
 Requires-Dist: websockets
 Requires-Dist: torch
 Requires-Dist: librosa
+Requires-Dist: pycountry
 Provides-Extra: dev
 Requires-Dist: pytest==7.4.0; extra == "dev"
 Requires-Dist: flake8; extra == "dev"
@@ -226,7 +227,7 @@ Provides-Extra: hf
 Requires-Dist: transformers==4.48.1; extra == "hf"
 Provides-Extra: canary
 Requires-Dist: Cython; extra == "canary"
-Requires-Dist: nemo_toolkit[asr]==2.4.0; extra == "canary"
+Requires-Dist: nemo_toolkit[asr]==3.0.0; extra == "canary"
 Provides-Extra: vad
 Requires-Dist: silero-vad; extra == "vad"
 Provides-Extra: eval

{simulstream-0.3.0 → simulstream-1.0.0}/pyproject.toml

@@ -14,7 +14,8 @@ dependencies = [
     "pyyaml>6.0",
     "websockets",
     "torch",
-    "librosa"
+    "librosa",
+    "pycountry"
 ]
 dynamic = ["version"]
 
@@ -52,7 +53,7 @@ hf = [
 
 canary = [
     "Cython",
-    "nemo_toolkit[asr]==2.4.0",
+    "nemo_toolkit[asr]==3.0.0",
 ]
 
 vad = [

simulstream-1.0.0/simulstream/VERSION.txt

@@ -0,0 +1 @@
+1.0.0

{simulstream-0.3.0 → simulstream-1.0.0}/simulstream/client/wav_reader_client.py

@@ -159,7 +159,11 @@ def load_wav_file_list(list_file_path: str) -> List[str]:
     """
     basedir = os.path.dirname(list_file_path)
     with open(list_file_path, 'r') as f:
-        wav_files = [basedir + '/' + line.strip() for line in f if line.strip()]
+        wav_files = [
+            line.strip() if os.path.isabs(line.strip())
+            else os.path.abspath(os.path.join(basedir, line.strip()))
+            for line in f if line.strip()
+        ]
     if not wav_files:
         LOGGER.error("No valid WAV files found in the list.")
         exit(1)

{simulstream-0.3.0 → simulstream-1.0.0}/simulstream/metrics/scorers/latency/mwersegmenter.py

@@ -123,7 +123,7 @@ class MWERSegmenterBasedLatencyScorer(LatencyScorer):
                     encoded = [" ".join(self.segmenter.encode(p)) for p in pieces]
                     tokenized_text.append(" ### ".join(encoded))
                 else:
-                    tokenized_text.append(" ".join(self.segmenter.encode(text[i].strip())))
+                    tokenized_text.append(" ".join(self.segmenter.encode(text[i])))
             return "\n".join(tokenized_text)
         else:
             return "\n".join(text)

simulstream-1.0.0/simulstream/server/speech_processors/base_doa.py

@@ -0,0 +1,230 @@
+# Copyright 2026 FBK
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+
+import logging
+from abc import abstractmethod
+from types import SimpleNamespace
+from typing import List, Tuple, Any, Dict
+
+import numpy as np
+import pycountry
+import torch
+
+from simulstream.server.speech_processors import SAMPLE_RATE
+from simulstream.server.speech_processors.base_streamatt import BaseStreamAtt
+
+
+logger = logging.getLogger(__name__)
+
+
+def get_language_name(code: str) -> str:
+    """Return the language name for an ISO 639-1 code, falling back to the code itself."""
+    lang = pycountry.languages.get(alpha_2=code)
+    if lang is not None:
+        return lang.name
+    else:
+        logger.warning(f"Language code '{code}' not found in the language list. Using language "
+                       f"code directly in the prompt, but this can lead to unexpected behavior.")
+        return code
+
+
+class DecoderOnlyAttention(BaseStreamAtt):
+    """
+    Generic Decoder-only Attention-based policy for SpeechLLMs.
+
+    The class handles:
+       - Raw-waveform history accumulation.
+       - Greedy generation with ``output_attentions=True``.
+       - Building the proxy cross-attention matrix from self-attention weights.
+       - Applying the StreamAtt-based policy on the proxy cross-attention matrix.
+
+    The derived class should implement the following methods:
+        - **load_model**: Loads the model and processor.
+        - **build_prompt**: Builds the text prompt to use with audio inputs.
+        - **build_processor_inputs**: Builds model inputs from the rolling audio history.
+        - **_do_generate**: Returns newly-generated tokens and self-attention scores.
+        - **_find_audio_positions**: Returns the indices of audio tokens.
+
+    Args:
+       config (SimpleNamespace): Configuration object. The following additional attributes are
+           expected:
+           - **attn_layer (int)**: Layer from which to extract attention scores. Defaults to 0.
+           - **attn_head (int)**: Attention head to use. If not set, attention scores are averaged
+             over all heads.
+           - **average_attn_over_layers (bool)**: Whether to average the selected attention view
+             over all decoder layers. Defaults to True.
+           - **audio_history_max_duration (int)**: Maximum raw waveform length to keep in the
+             rolling history, in seconds. Defaults to 180.
+           - **max_new_tokens (int)**: Maximum tokens to generate per chunk. Defaults to 32.
+    """
+
+    def __init__(self, config: SimpleNamespace):
+        super().__init__(config)
+        self.cross_attn_layer = getattr(self.config, "attn_layer", 0)
+        self.cross_attn_head = getattr(self.config, "attn_head", None)
+        self.average_attn_over_layers = getattr(self.config, "average_attn_over_layers", True)
+        self.audio_history_max_duration = getattr(self.config, "audio_history_max_duration", 180)
+        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        self.max_new_tokens = getattr(self.config, "max_new_tokens", 32)
+        self.prompt = getattr(self.config, "prompt", "Translate the audio to {tgt_lang}:")
+        logger.debug("Prompt:\n%s", self.prompt)
+
+    @property
+    def audio_max_len(self) -> int:
+        """Maximum raw-waveform samples to keep in the rolling audio history."""
+        return self.audio_history_max_duration * SAMPLE_RATE
+
+    @abstractmethod
+    def load_model(self, config: SimpleNamespace) -> None:
+        """
+        Load the model and processor from *config* and assign them to ``self.model`` and
+        ``self.processor``.
+
+        The model **must** be loaded with ``output_attentions=True`` (or the equivalent flag for
+        the architecture) and ``_attn_implementation="eager"``.
+        """
+        ...
+
+    @abstractmethod
+    def build_prompt(self) -> str:
+        """
+        Return the prompt string to be used with audio tokens.
+        """
+        ...
+
+    @abstractmethod
+    def build_processor_inputs(self, waveform: np.ndarray) -> Any:
+        """
+        Build processor inputs from the entire rolling waveform history (float32, 16 kHz).
+        """
+        ...
+
+    @abstractmethod
+    def _do_generate(self, inputs: Dict[str, Any]) -> Tuple[List[str], List[torch.Tensor]]:
+        """
+        Runs the actual generation from the underlying model and returns the generated tokens and
+        the corresponding self-attention scores.
+        """
+        ...
+
+    @abstractmethod
+    def _find_audio_positions(self, input_ids: torch.Tensor) -> torch.Tensor:
+        """Return token positions corresponding to the encoded audio span."""
+        ...
+
+    def _generate(self, waveform: np.ndarray) -> Tuple[List[str], torch.Tensor]:
+        """
+        Generate tokens from the given inputs together with the self-attention scores.
+
+        Returns:
+            Tuple[List[str], torch.Tensor]:
+                List[str]: A list of generated tokens.
+                torch.Tensor: Self-attention scores between speech and text with dimension
+                (token_len, audio_len).
+        """
+        inputs = self.build_processor_inputs(waveform).to(self.device)
+        input_ids = inputs["input_ids"]  # (1, input_len)
+        input_len = input_ids.shape[1]
+
+        audio_positions = self._find_audio_positions(input_ids)
+        audio_len = audio_positions.shape[0]
+
+        # Run the actual generate on the underlying model
+        new_tokens, attentions = self._do_generate(inputs)
+
+        # Build proxy cross-attention for the hypothesis (prefix + new_tokens)
+        prefill_attn = self.average_attn(attentions[0])
+        prefix_len = len(self.text_history) if self.text_history else 0
+        if prefix_len > 0:
+            # Prefix rows come from the prefill pass
+            prefix_rows = prefill_attn[input_len - prefix_len:, :][:, audio_positions]
+        else:
+            prefix_rows = torch.zeros(0, audio_len, device=self.device)
+
+        if new_tokens:
+            # The prefill pass predicts the first generated token, so its last row corresponds to
+            # the first generated token's proxy audio-attention
+            first_new_row = prefill_attn[-1:, audio_positions]
+            # Other tokens' attention is present in each generation step
+            new_rows = [
+                self.average_attn(step_attn).squeeze(0)[audio_positions]
+                for step_attn in attentions[1:]
+            ]
+            subsequent_new_attn = torch.stack(new_rows, dim=0)
+            new_attn = torch.cat([first_new_row, subsequent_new_attn], dim=0)
+        else:
+            new_attn = torch.zeros(0, audio_len, device=self.device)
+
+        cross_attn = torch.cat([prefix_rows, new_attn], dim=0)
+        cross_attn = self.normalize_attn(cross_attn)
+        return new_tokens, cross_attn
+
+    def set_target_language(self, language: str) -> None:
+        self.tgt_lang = language
+
+    def set_source_language(self, language: str) -> None:
+        self.src_lang = language
+
+    def build_raw_text_prefix(self) -> str:
+        return "".join(self.text_history) if self.text_history else ""
+
+    def _select_attn_from_layer(self, layer_attn: torch.Tensor) -> torch.Tensor:
+        # Generation runs one stream at a time, so remove the singleton batch dimension.
+        layer_attn = layer_attn.squeeze(0)
+        if self.cross_attn_head is None:
+            # Default behavior: average over all heads for this layer.
+            return layer_attn.mean(dim=0)
+
+        num_heads = layer_attn.shape[0]
+        if self.cross_attn_head < 0 or self.cross_attn_head >= num_heads:
+            raise ValueError(
+                f"Invalid attn_head={self.cross_attn_head}. Layer has {num_heads} heads."
+            )
+        return layer_attn[self.cross_attn_head]
+
+    def average_attn(self, attn) -> torch.Tensor:
+        """
+        Average or select attentions according to ``attn_layer``, ``attn_head``, and
+        ``average_attn_over_layers``.
+
+        If ``attn_head`` is not set, attention is averaged over heads. If
+        ``average_attn_over_layers`` is set, the selected per-layer attention view is also
+        averaged across layers; otherwise only ``attn_layer`` is used.
+        """
+        if self.average_attn_over_layers:
+            # Average the per-layer attention view selected by _select_attn_from_layer.
+            return torch.stack(
+                [self._select_attn_from_layer(layer_attn) for layer_attn in attn],
+                dim=0,
+            ).mean(dim=0)
+        return self._select_attn_from_layer(attn[self.cross_attn_layer])
+
+    def _preprocess(self, waveform: np.float32) -> np.ndarray:
+        """
+        Append *waveform* to ``self.audio_history`` and enforce the maximum length.
+        """
+        if self.audio_history is None:
+            self.audio_history = waveform
+        else:
+            self.audio_history = np.concatenate([self.audio_history, waveform])
+
+        if len(self.audio_history) > self.audio_max_len:
+            logger.warning("Audio history exceeded %d samples; trimming.", self.audio_max_len)
+            self.audio_history = self.audio_history[-self.audio_max_len:]
+
+        return self.audio_history
+
+    def tokens_to_string(self, tokens: List[str]) -> str:
+        """Convert a list of decoded tokens to a plain output string."""
+        return "".join(tokens)

{simulstream-0.3.0 → simulstream-1.0.0}/simulstream/server/speech_processors/base_streamatt.py

@@ -60,6 +60,10 @@ class BaseStreamAtt(BaseSpeechProcessor):
                  context for next predictions.
            - **audio_subsampling_factor (int)**: Subsampling factor of the model, if any.
              Defaults to 1.
+           - **mel_hop_samples (int)**: Number of raw waveform samples per mel frame.
+             Defaults to 160, i.e. 10ms at 16kHz.
+           - **use_raw_audio_history (bool)**: Returns whether ``audio_history`` stores raw
+             waveform samples rather than processed frames. Defaults to False.
            - **text_history_max_len (int)**: The maximum length of the textual history after which
              the current content is cut. Defaults to 128.
            - **cross_attention_layer (int)**: Layer from which to extract the cross-attention from.
@@ -75,8 +79,14 @@ class BaseStreamAtt(BaseSpeechProcessor):
         self.config = config
         text_history_config = self.config.text_history
         text_history_cls = class_load(text_history_config.type)
+        self.bow_prefix = getattr(self.config, "bow_prefix", BOW_PREFIX)
         self.text_history_method = text_history_cls(text_history_config)
         self.audio_subsampling_factor = getattr(self.config, "audio_subsampling_factor", 1)
+        self.mel_hop_samples = getattr(self.config, "mel_hop_samples", 160)
+        self.use_raw_audio_history = getattr(self.config, "use_raw_audio_history", False)
+        self.frames_to_audio_history = self.audio_subsampling_factor
+        if self.use_raw_audio_history:
+            self.frames_to_audio_history *= self.mel_hop_samples
         self.text_history_max_len = getattr(self.config, "text_history_max_len", 128)
         self.cross_attn_layer = getattr(self.config, "cross_attention_layer", 3)
         self.cutoff_frame_num = getattr(self.config, "cutoff_frame_num", 2)
@@ -173,8 +183,8 @@ class BaseStreamAtt(BaseSpeechProcessor):
             # Only one token: use the unique most attended frame
             earliest_attended_idx = most_attended_idxs[0]
 
-        # Multiply by the subsampling factor to recover the original number of frames
-        frames_to_cut = earliest_attended_idx * self.audio_subsampling_factor
+        # Multiply by the number of frames/samples corresponding to the audio history
+        frames_to_cut = earliest_attended_idx * self.frames_to_audio_history
 
         # Cut the unattended audio features
         self.audio_history = self.audio_history[frames_to_cut:]
@@ -182,8 +192,7 @@ class BaseStreamAtt(BaseSpeechProcessor):
         # Check audio history not exceeding maximum allowed length
         self._cut_audio_exceeding_maxlen()
 
-    @staticmethod
-    def _strip_incomplete_words(tokens: List[str]) -> List[str]:
+    def _strip_incomplete_words(self, tokens: List[str]) -> List[str]:
         """
         Remove last incomplete word(s) from the new hypothesis.
 
@@ -198,7 +207,7 @@ class BaseStreamAtt(BaseSpeechProcessor):
         num_tokens_incomplete = 0
         for tok in reversed(tokens):
             num_tokens_incomplete += 1
-            if tok.startswith(BOW_PREFIX):
+            if tok.startswith(self.bow_prefix):
                 # slice off the trailing incomplete tokens
                 tokens_to_write = tokens[:-num_tokens_incomplete]
                 break
@@ -273,11 +282,10 @@ class FixedWordsTextHistory:
     """
     Fixed Words textual history selection method that retains a pre-defined
     number of words in the history (*history_words*).
-
-    The current implementation supports only SentencePiece.
     """
     def __init__(self, config: SimpleNamespace):
         self.history_words = getattr(config, "history_words", 20)
+        self.bow_prefix = getattr(config, "bow_prefix", BOW_PREFIX)
         self.config = config
 
     def select_text_history(self, text_history: List[str]):
@@ -285,9 +293,9 @@ class FixedWordsTextHistory:
         new_history = []
         for token in reversed(text_history):
             new_history.append(token)
-            # Check if 'BOW_PREFIX' (space in SentencePiece) is contained in the token,
-            # meaning that we reached the beginning of the word that should be counted
-            if BOW_PREFIX in token:
+            # Check if bow_prefix is contained in the token, meaning that we reached
+            # the beginning of the word that should be counted
+            if self.bow_prefix in token:
                 words_to_keep -= 1
                 # When all the words to keep are consumed, the accumulation is stopped
                 # and the prefix is returned
@@ -297,6 +305,27 @@ class FixedWordsTextHistory:
         return new_history[::-1]
 
 
+class FixedCharsTextHistory:
+    """
+    Character-count-based textual history selection method that retains a pre-defined number of
+    tokens in the history (*history_chars*).
+
+    Recommended for character-level languages (e.g., Chinese, Japanese) where word-boundary
+    markers (▁) are sparse, making :class:`FixedWordsTextHistory` ineffective: when few tokens
+    carry a BOW prefix, the word counter never reaches *history_words*, so the history is never
+    trimmed and the audio history grows without bound, causing AlignAtt to cut all new tokens.
+
+    Args:
+        config (SimpleNamespace): Configuration object with an optional attribute:
+            - **history_chars (int)**: Number of tokens to retain. Defaults to 20.
+    """
+    def __init__(self, config: SimpleNamespace):
+        self.history_chars = getattr(config, "history_chars", 20)
+
+    def select_text_history(self, text_history: List[str]) -> List[str]:
+        return text_history[-self.history_chars:]
+
+
 class PunctuationTextHistory:
     """
     Punctuation textual history selection method that retains the sentence

simulstream-1.0.0/simulstream/server/speech_processors/canary_streamatt.py

@@ -0,0 +1,158 @@
+# Copyright 2025 FBK
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+
+import torch
+import numpy as np
+
+from types import SimpleNamespace
+from typing import List, Tuple
+
+import copy
+
+from simulstream.server.speech_processors import SAMPLE_RATE
+from simulstream.server.speech_processors.base_streamatt import BaseStreamAtt
+
+from nemo.collections.asr.models import ASRModel
+from nemo.collections.asr.parts.submodules.multitask_decoding import (
+    MultiTaskDecodingConfig,
+)
+from nemo.collections.asr.models.aed_multitask_models import (
+    MultiTaskTranscriptionConfig,
+)
+
+
+class CanaryStreamAtt(BaseStreamAtt):
+    """
+    StreamAtt policy implementation for NVIDIA's Canary-v2 model.
+
+    Args:
+        config (SimpleNamespace): Configuration object.
+            Supported attributes:
+            - **audio_history_max_duration (int)**: Maximum audio history in seconds.
+              Defaults to ``30``.
+            - **num_beams (int)**: Number of beams to use for beam search decoding.
+              Defaults to ``5``.
+    """
+
+    def __init__(self, config: SimpleNamespace):
+        super().__init__(config)
+        self._audio_history_max_duration = getattr(self.config, "audio_history_max_duration", 30)
+
+        expected_mel_hop_samples = (
+            self.model.cfg.preprocessor.window_stride * self.model.cfg.preprocessor.sample_rate
+        )
+
+        assert self.mel_hop_samples == expected_mel_hop_samples, (
+            f"mel_hop_samples is set to {self.mel_hop_samples} in the config, but the loaded "
+            f"model's preprocessor uses {expected_mel_hop_samples} samples per mel frame"
+        )
+
+        # Build the transcription config, which is reused for every transcribe() call.
+        self.transcription_cfg = MultiTaskTranscriptionConfig(
+            batch_size=1,
+            return_hypotheses=True,
+            enable_chunking=False,
+            verbose=False,
+        )
+
+    @property
+    def audio_max_len(self) -> int:
+        """Maximum audio history length in raw waveform samples."""
+        return self._audio_history_max_duration * SAMPLE_RATE
+
+    def set_source_language(self, language: str) -> None:
+        self.src_lang = language
+
+    def set_target_language(self, language: str) -> None:
+        self.tgt_lang = language
+
+    @classmethod
+    def load_model(cls, config: SimpleNamespace):
+        if not hasattr(cls, "model") or cls.model is None:
+            cls.model = ASRModel.from_pretrained(model_name=config.model_name)
+
+            # Configure decoding strategy
+            multitask_decoding = MultiTaskDecodingConfig()
+            multitask_decoding.strategy = "beam"
+            multitask_decoding.return_xattn_scores = True
+            multitask_decoding.beam.beam_size = getattr(config, "num_beams", 5)
+            cls.model.change_decoding_strategy(multitask_decoding)
+
+            cls.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+            assert cls.model.cfg.preprocessor.sample_rate == SAMPLE_RATE
+            cls.model.to(cls.device)
+
+    def _build_transcription_config(self):
+        """
+        Return a ``MultiTaskTranscriptionConfig`` whose prompt encodes the current source/target
+        languages, task, PNC preference, and forced decoder prefix.
+        """
+
+        default_turns = self.model.prompt.get_default_dialog_slots()
+        default_slots = copy.deepcopy(default_turns[0]["slots"])
+        default_slots["source_lang"] = self.src_lang
+        default_slots["target_lang"] = self.tgt_lang
+
+        turns = [
+            {
+                "role": "user", "slots": default_slots
+            },
+            {
+                "role": "user_prefix",
+                "slots": {
+                    "prefix": self.model.tokenizer.tokens_to_text(self.text_history)
+                },
+            },
+        ]
+
+        cfg_copy = copy.deepcopy(self.transcription_cfg)
+        cfg_copy.prompt = turns
+
+        return cfg_copy
+
+    def _preprocess(self, waveform: np.ndarray) -> np.ndarray:
+        """
+        Append the incoming waveform chunk to the raw audio history and return it.
+
+        Returns:
+            np.ndarray: Accumulated raw audio history.
+        """
+        waveform = waveform.astype(np.float32)
+        if self.audio_history is None:
+            self.audio_history = waveform
+        else:
+            self.audio_history = np.concatenate(
+                [self.audio_history, waveform])
+
+        return self.audio_history
+
+    def _generate(self, speech: np.ndarray) -> Tuple[List[str], torch.Tensor]:
+        override_config = self._build_transcription_config()
+
+        with torch.inference_mode():
+            output = self.model.transcribe(audio=speech, override_config=override_config)
+
+        hypothesis = output[0]
+
+        token_ids = hypothesis.y_sequence.detach().cpu().tolist()
+        tokens = self.model.tokenizer.ids_to_tokens(token_ids)
+
+        xatt_raw = hypothesis.xatt_scores[self.cross_attn_layer]
+        xatt = xatt_raw.mean(dim=0).cpu()  # we average over heads
+        xatt = self.normalize_attn(xatt)
+
+        return tokens, xatt
+
+    def tokens_to_string(self, tokens: List[str]) -> str:
+        return self.model.tokenizer.tokens_to_text(tokens)

simulstream-1.0.0/simulstream/server/speech_processors/phi4multimodal_doa.py

@@ -0,0 +1,97 @@
+# Copyright 2026 FBK
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+
+from types import SimpleNamespace
+from typing import List, Any, Dict, Tuple
+
+import numpy as np
+import torch
+from transformers import AutoModelForCausalLM, AutoProcessor, GenerationConfig
+
+from simulstream.server.speech_processors import SAMPLE_RATE
+from simulstream.server.speech_processors.base_doa import DecoderOnlyAttention, get_language_name
+
+
+class Phi4MultimodalDOA(DecoderOnlyAttention):
+    """
+    Decoder-Only Attention agent for Phi4-Multimodal.
+    """
+
+    # Phi-4 special tokens
+    _USER_START = "<|user|>"
+    _AUDIO_TOKEN = "<|audio_1|>"
+    _END_TOKEN = "<|end|>"
+    _ASST_START = "<|assistant|>"
+
+    ENCODER_SUBSAMPLING_FACTOR = 8
+    HOP_LENGTH = 160  # 10ms at 16kHz
+    AUDIO_SPECIAL_TOKEN_ID = 200011  # _AUDIO_SPECIAL_TOKEN_ID in modeling_phi4mm.py
+
+    def __init__(self, config: SimpleNamespace):
+        super().__init__(config)
+        self.audio_subsampling_factor = self.ENCODER_SUBSAMPLING_FACTOR * self.HOP_LENGTH
+
+    @classmethod
+    def load_model(cls, config: SimpleNamespace) -> None:
+        model_path = getattr(
+            config,
+            "hf_model_name",
+            getattr(config, "model_path", "microsoft/Phi-4-multimodal-instruct"),
+        )
+
+        cls.processor = AutoProcessor.from_pretrained(model_path, trust_remote_code=True)
+        cls.model = AutoModelForCausalLM.from_pretrained(
+            model_path,
+            device_map="cuda",
+            torch_dtype="auto",
+            trust_remote_code=True,
+            _attn_implementation="eager",
+        )
+        cls.model.eval()
+        cls.generation_config = GenerationConfig.from_pretrained(model_path)
+
+    def build_prompt(self) -> str:
+        filled_prompt = self.prompt.replace("{tgt_lang}", get_language_name(self.tgt_lang))
+        raw_prefix = self.build_raw_text_prefix()
+        prompt = (
+            f"{self._USER_START}{self._AUDIO_TOKEN}"
+            f"{filled_prompt}{self._END_TOKEN}"
+            f"{self._ASST_START}{raw_prefix}"
+        )
+        return prompt
+
+    def build_processor_inputs(self, waveform: np.ndarray) -> dict:
+        return self.processor(
+            text=self.build_prompt(),
+            audios=[(waveform, SAMPLE_RATE)],
+            return_tensors="pt",
+        )
+
+    def _do_generate(self, inputs: Dict[str, Any]) -> Tuple[List[str], List[torch.Tensor]]:
+        input_len = inputs["input_ids"].shape[1]
+        output = self.model.generate(
+            **inputs,
+            max_new_tokens=self.max_new_tokens,
+            generation_config=self.generation_config,
+            num_logits_to_keep=1,
+            output_attentions=True,
+            return_dict_in_generate=True,
+            do_sample=False,
+        )
+        new_tokens = self.processor.tokenizer.convert_ids_to_tokens(
+            output.sequences[0, input_len:], skip_special_tokens=True)
+        return new_tokens, output.attentions
+
+    def _find_audio_positions(self, input_ids: torch.Tensor) -> torch.Tensor:
+        return (input_ids[0] == self.AUDIO_SPECIAL_TOKEN_ID).nonzero(as_tuple=True)[0]