visqol-python 3.3.3__py3-none-any.whl

visqol/nsim.py

@@ -0,0 +1,134 @@
+"""
+Neurogram Similarity Index Measure (NSIM).
+
+A variant of SSIM adapted for neurogram (spectrogram) comparison.
+
+Corresponds to C++ files:
+  - neurogram_similiarity_index_measure.cc
+  - convolution_2d.cc
+"""
+
+import numpy as np
+from dataclasses import dataclass, field
+
+
+# 3x3 Gaussian window weights (hardcoded from C++)
+GAUSSIAN_WINDOW = np.array([
+    [0.0113033910173052, 0.0838251475442633, 0.0113033910173052],
+    [0.0838251475442633, 0.619485845753726,  0.0838251475442633],
+    [0.0113033910173052, 0.0838251475442633, 0.0113033910173052]
+])
+
+# Constants for NSIM calculation
+INTENSITY_RANGE = 1.0
+K1 = 0.01
+K2 = 0.03
+C1 = (K1 * INTENSITY_RANGE) ** 2  # = 0.0001
+C3 = (K2 * INTENSITY_RANGE) ** 2 / 2.0  # = 0.00045
+
+
+@dataclass
+class PatchSimilarityResult:
+    """Result of comparing a reference patch with a degraded patch."""
+    similarity: float = 0.0
+    freq_band_means: np.ndarray = field(default_factory=lambda: np.array([]))
+    freq_band_stddevs: np.ndarray = field(default_factory=lambda: np.array([]))
+    freq_band_deg_energy: np.ndarray = field(default_factory=lambda: np.array([]))
+
+    # Timing info
+    ref_patch_start_time: float = 0.0
+    ref_patch_end_time: float = 0.0
+    deg_patch_start_time: float = 0.0
+    deg_patch_end_time: float = 0.0
+
+
+def _valid_2d_conv_with_boundary(kernel: np.ndarray,
+                                  matrix: np.ndarray) -> np.ndarray:
+    """
+    2D convolution with boundary replication padding, then 'valid' convolution.
+
+    Matches C++ Convolution2D::Valid2DConvWithBoundary which:
+    1. Pads matrix by 1 on each side with edge replication
+    2. Performs valid convolution with REVERSED kernel
+
+    The C++ code reverses the kernel in column-major order during convolution.
+    Since our Gaussian kernel is symmetric (both row-symmetric and column-symmetric),
+    the reversal has no effect. We can use scipy's fast convolution directly.
+
+    The output has the same shape as the input matrix (edge-padded then valid conv).
+    """
+    from scipy.ndimage import convolve
+
+    # Pad matrix by 1 on each side with edge replication
+    padded = np.pad(matrix, pad_width=1, mode='edge')
+
+    # The C++ reverses the kernel in column-major layout.
+    # For the symmetric Gaussian kernel, this is equivalent to no reversal.
+    # Use scipy.ndimage.convolve which does correlation (no kernel flip)
+    # on the padded matrix, then take the valid region.
+
+    # scipy.ndimage.convolve handles the full convolution;
+    # we use mode='constant' with cval=0 since we already padded.
+    # But more efficiently: just use 'valid' equivalent by slicing.
+    from scipy.signal import correlate2d
+    result = correlate2d(padded, kernel, mode='valid')
+    return result
+
+
+def measure_patch_similarity(ref_patch: np.ndarray,
+                              deg_patch: np.ndarray) -> PatchSimilarityResult:
+    """
+    Compute NSIM similarity between a reference and degraded patch.
+
+    Matches C++ NeurogramSimiliarityIndexMeasure::MeasurePatchSimilarity.
+
+    Args:
+        ref_patch: (num_bands, num_frames) reference spectrogram patch.
+        deg_patch: (num_bands, num_frames) degraded spectrogram patch.
+
+    Returns:
+        PatchSimilarityResult with similarity score and per-band statistics.
+    """
+    w = GAUSSIAN_WINDOW
+
+    # Local means
+    mu_r = _valid_2d_conv_with_boundary(w, ref_patch)
+    mu_d = _valid_2d_conv_with_boundary(w, deg_patch)
+
+    # Squared means
+    ref_mu_sq = mu_r * mu_r
+    deg_mu_sq = mu_d * mu_d
+    mu_r_mu_d = mu_r * mu_d
+
+    # Variances
+    sigma_r_sq = _valid_2d_conv_with_boundary(w, ref_patch * ref_patch) - ref_mu_sq
+    sigma_d_sq = _valid_2d_conv_with_boundary(w, deg_patch * deg_patch) - deg_mu_sq
+    sigma_r_d = _valid_2d_conv_with_boundary(w, ref_patch * deg_patch) - mu_r_mu_d
+
+    # Intensity (luminance) component
+    intensity = (2.0 * mu_r_mu_d + C1) / (ref_mu_sq + deg_mu_sq + C1)
+
+    # Structure component
+    structure_numer = sigma_r_d + C3
+    var_product = sigma_r_sq * sigma_d_sq
+    # Handle negative variance (can occur with silent patches)
+    structure_denom = np.where(var_product < 0, C3, np.sqrt(var_product) + C3)
+    structure = structure_numer / structure_denom
+
+    # Combined similarity map
+    sim_map = intensity * structure
+
+    # Per-frequency-band statistics
+    freq_band_means = np.mean(sim_map, axis=1)
+    freq_band_stddevs = np.std(sim_map, axis=1, ddof=0)
+    freq_band_deg_energy = np.mean(deg_patch, axis=1)
+
+    # Overall similarity (mean of frequency band means)
+    mean_freq_band_means = np.mean(freq_band_means)
+
+    return PatchSimilarityResult(
+        similarity=float(mean_freq_band_means),
+        freq_band_means=freq_band_means,
+        freq_band_stddevs=freq_band_stddevs,
+        freq_band_deg_energy=freq_band_deg_energy,
+    )

visqol/patch_creator.py

@@ -0,0 +1,222 @@
+"""
+Patch creation for reference spectrograms.
+- ImagePatchCreator: evenly-spaced patches (Audio mode)
+- VadPatchCreator: VAD-filtered patches (Speech mode)
+
+Corresponds to C++ files:
+  - image_patch_creator.cc
+  - vad_patch_creator.cc
+  - rms_vad.cc/h
+"""
+
+import numpy as np
+from visqol.analysis_window import AnalysisWindow
+from visqol.audio_utils import AudioSignal
+from visqol.signal_utils import normalize
+
+
+# ============ RMS VAD ============
+
+class RmsVad:
+    """
+    Simple RMS-based Voice Activity Detection.
+    Matches C++ RmsVad class.
+    """
+    VOICE_ACTIVITY_PRESENT = 1.0
+    VOICE_ACTIVITY_ABSENT = 0.0
+    SILENT_CHUNK_COUNT = 3
+    RMS_THRESHOLD = 5000.0
+
+    def __init__(self):
+        self.each_chunk_result = []
+        # Initialize first (kSilentChunkCount - 1) results as voice-active
+        # to avoid false negatives
+        self.vad_results = [self.VOICE_ACTIVITY_PRESENT] * (self.SILENT_CHUNK_COUNT - 1)
+
+    def process_chunk(self, chunk: np.ndarray) -> float:
+        """Process a chunk of int16 samples."""
+        rms = np.sqrt(np.mean(chunk.astype(np.float64) ** 2))
+        if rms < self.RMS_THRESHOLD:
+            self.each_chunk_result.append(self.VOICE_ACTIVITY_ABSENT)
+        else:
+            self.each_chunk_result.append(self.VOICE_ACTIVITY_PRESENT)
+        return rms
+
+    def get_vad_results(self) -> list:
+        """Get VAD results for all processed chunks."""
+        for i in range(self.SILENT_CHUNK_COUNT - 1, len(self.each_chunk_result)):
+            if (not self.each_chunk_result[i] and
+                    self._check_previous_chunks_for_silence(i)):
+                self.vad_results.append(self.VOICE_ACTIVITY_ABSENT)
+            else:
+                self.vad_results.append(self.VOICE_ACTIVITY_PRESENT)
+        return self.vad_results
+
+    def _check_previous_chunks_for_silence(self, idx: int) -> bool:
+        """Check if previous chunks are also silent."""
+        for j in range(1, self.SILENT_CHUNK_COUNT):
+            if self.each_chunk_result[idx - j] == self.VOICE_ACTIVITY_PRESENT:
+                return False
+        return True
+
+
+# ============ Image Patch Creator (Audio mode) ============
+
+class ImagePatchCreator:
+    """
+    Creates evenly-spaced patches from a spectrogram.
+    Used for Audio mode.
+    """
+
+    def __init__(self, patch_size: int):
+        self.patch_size = patch_size
+
+    def create_ref_patch_indices(self, spectrogram: np.ndarray,
+                                 ref_signal: AudioSignal = None,
+                                 window: AnalysisWindow = None) -> list:
+        """
+        Create reference patch indices at evenly-spaced intervals.
+
+        Matches C++ ImagePatchCreator::CreateRefPatchIndices.
+
+        Args:
+            spectrogram: (num_bands, num_frames) spectrogram matrix.
+
+        Returns:
+            List of start column indices for patches.
+        """
+        spectrum_length = spectrogram.shape[1]
+        init_patch_index = self.patch_size // 2
+
+        if spectrum_length < self.patch_size + init_patch_index:
+            raise ValueError(
+                f"Reference spectrum size ({spectrum_length}) smaller than "
+                f"minimum patch size ({self.patch_size + init_patch_index})."
+            )
+
+        # C++: max_index logic
+        if init_patch_index < (spectrum_length - self.patch_size):
+            max_index = spectrum_length - self.patch_size
+        else:
+            max_index = init_patch_index + 1
+
+        indices = []
+        i = init_patch_index
+        while i < max_index:
+            indices.append(i - 1)  # C++ uses 0-based, pushes i-1
+            i += self.patch_size
+
+        return indices
+
+    def create_patches_from_indices(self, spectrogram: np.ndarray,
+                                     patch_indices: list) -> list:
+        """
+        Extract patches from spectrogram at given start indices.
+
+        Args:
+            spectrogram: (num_bands, num_frames) matrix.
+            patch_indices: List of start column indices.
+
+        Returns:
+            List of (num_bands, patch_size) patch matrices.
+        """
+        patches = []
+        for start_col in patch_indices:
+            end_col = start_col + self.patch_size
+            patch = spectrogram[:, start_col:end_col]
+            patches.append(patch)
+        return patches
+
+
+# ============ VAD Patch Creator (Speech mode) ============
+
+class VadPatchCreator:
+    """
+    Creates VAD-filtered patches from a spectrogram.
+    Used for Speech mode.
+    """
+
+    FRAMES_WITH_VA_THRESHOLD = 1.0
+
+    def __init__(self, patch_size: int):
+        self.patch_size = patch_size
+
+    def _get_voice_activity(self, signal: AudioSignal, start_sample: int,
+                            total_samples: int, frame_len: int) -> list:
+        """
+        Get voice activity detection results for a signal segment.
+
+        Matches C++ VadPatchCreator::GetVoiceActivity.
+        """
+        rms_vad = RmsVad()
+        data = signal.data[start_sample:start_sample + total_samples]
+
+        # Convert to int16 range and process in chunks
+        frame = []
+        for val in data:
+            # Quantize to int16 range
+            int_val = val * (1 << 15)
+            int_val = max(-1.0 * (1 << 15),
+                          min(1.0 * ((1 << 15) - 1), int_val))
+            frame.append(int(int_val))
+            if len(frame) == frame_len:
+                rms_vad.process_chunk(np.array(frame, dtype=np.int16))
+                frame = []
+
+        return rms_vad.get_vad_results()
+
+    def create_ref_patch_indices(self, spectrogram: np.ndarray,
+                                 ref_signal: AudioSignal,
+                                 window: AnalysisWindow) -> list:
+        """
+        Create VAD-filtered reference patch indices.
+
+        Matches C++ VadPatchCreator::CreateRefPatchIndices.
+        """
+        # Normalize signal
+        norm_data = normalize(ref_signal.data)
+        norm_signal = AudioSignal(norm_data, ref_signal.sample_rate)
+
+        frame_size = int(window.size * window.overlap)
+        patch_sample_len = self.patch_size * frame_size
+        spectrum_length = spectrogram.shape[1]
+        first_patch_idx = self.patch_size // 2 - 1
+        patch_count = (spectrum_length - first_patch_idx) // self.patch_size
+        total_sample_count = patch_count * patch_sample_len
+
+        # Get VAD results
+        vad_res = self._get_voice_activity(
+            norm_signal, first_patch_idx, total_sample_count, frame_size
+        )
+
+        # Filter patches based on VAD
+        ref_patch_indices = []
+        patch_idx = first_patch_idx
+        for i in range(patch_count):
+            # Slice VAD data for this patch
+            start = i * self.patch_size
+            end = start + self.patch_size
+            if end <= len(vad_res):
+                patch_vad = vad_res[start:end]
+            else:
+                patch_vad = vad_res[start:]
+
+            # Count frames with voice activity
+            frames_with_va = sum(patch_vad)
+
+            if frames_with_va >= self.FRAMES_WITH_VA_THRESHOLD:
+                ref_patch_indices.append(patch_idx)
+
+            patch_idx += self.patch_size
+
+        return ref_patch_indices
+
+    def create_patches_from_indices(self, spectrogram: np.ndarray,
+                                     patch_indices: list) -> list:
+        """Extract patches from spectrogram at given indices."""
+        patches = []
+        for start_col in patch_indices:
+            end_col = start_col + self.patch_size
+            patch = spectrogram[:, start_col:end_col]
+            patches.append(patch)
+        return patches