echo-vector 0.1.1__py3-none-any.whl

echovector/embeddings/clap.py

@@ -0,0 +1,126 @@
+"""CLAP (Contrastive Language-Audio Pretraining) embedding backend."""
+
+from typing import cast
+
+import librosa
+import numpy as np
+import numpy.typing as npt
+
+from echovector.embeddings.base import EmbeddingBackend
+
+try:
+    import torch
+    from transformers import ClapModel, ClapProcessor
+
+    _CLAP_AVAILABLE = True
+except ImportError:  # pragma: no cover
+    _CLAP_AVAILABLE = False
+
+
+class ClapBackend(EmbeddingBackend):
+    """Embedding backend using the CLAP model from Hugging Face transformers.
+
+    Supports both audio and text embeddings in the same semantic space.
+    """
+
+    def __init__(
+        self,
+        model_name: str = "laion/clap-htsat-unfused",
+        device: str | None = None,
+    ) -> None:
+        """Initialize the CLAP backend.
+
+        Args:
+            model_name: The Hugging Face model identifier.
+            device: Device to run the model on (e.g., 'cpu', 'cuda').
+
+        Raises:
+            ImportError: If torch or transformers are not installed.
+        """
+        if not _CLAP_AVAILABLE:
+            raise ImportError(
+                "CLAP backend requires torch and transformers. "
+                "Install them with: pip install 'echo_vector[clap]'"
+            )
+        if device is None:
+            self.device = "cuda" if torch.cuda.is_available() else "cpu"
+        else:
+            self.device = device
+
+        self.processor = ClapProcessor.from_pretrained(model_name)
+        self.model = ClapModel.from_pretrained(model_name)
+        self.model.to(self.device)
+        self.model.eval()
+
+        # Determine embedding dimension from model config
+        self._embedding_dim = self.model.config.projection_dim
+
+    @property
+    def embedding_dim(self) -> int:
+        """Return the dimensionality of the generated embeddings."""
+        return cast("int", self._embedding_dim)
+
+    def _load_and_resample(self, path: str, target_sr: int) -> np.ndarray:
+        """Load audio and resample to the target sample rate."""
+        audio_array, _ = librosa.load(path, sr=target_sr, mono=True)
+        return audio_array
+
+    def embed_audio(self, audio_paths: list[str]) -> npt.NDArray[np.float32]:
+        """Embed a batch of audio files into the CLAP semantic space.
+
+        Args:
+            audio_paths: List of file paths to audio files.
+
+        Returns:
+            A numpy array of shape (batch_size, embedding_dim).
+        """
+        target_sr = 48000  # Default for CLAP
+        feature_extractor = getattr(self.processor, "feature_extractor", None)
+        if feature_extractor is not None and hasattr(feature_extractor, "sampling_rate"):
+            target_sr = int(getattr(feature_extractor, "sampling_rate", target_sr))
+
+        audios = [self._load_and_resample(path, target_sr) for path in audio_paths]
+
+        inputs = self.processor(
+            audios=audios,
+            return_tensors="pt",
+            sampling_rate=target_sr,
+        )
+        inputs = {k: v.to(self.device) for k, v in inputs.items()}
+
+        with torch.no_grad():
+            audio_features = self.model.get_audio_features(**inputs)
+            # Normalize to ensure audio and text embeddings share the unit hypersphere
+            audio_features = audio_features / audio_features.norm(dim=-1, keepdim=True)
+
+        return cast(
+            "npt.NDArray[np.float32]",
+            audio_features.cpu().numpy().astype(np.float32),
+        )
+
+    def embed_text(self, texts: list[str]) -> npt.NDArray[np.float32]:
+        """Embed a batch of text queries into the CLAP semantic space.
+
+        Args:
+            texts: List of text strings.
+
+        Returns:
+            A numpy array of shape (batch_size, embedding_dim).
+        """
+        inputs = self.processor(
+            text=texts,
+            return_tensors="pt",
+            padding=True,
+            truncation=True,
+        )
+        inputs = {k: v.to(self.device) for k, v in inputs.items()}
+
+        with torch.no_grad():
+            text_features = self.model.get_text_features(**inputs)
+            # Normalize to ensure audio and text embeddings share the unit hypersphere
+            text_features = text_features / text_features.norm(dim=-1, keepdim=True)
+
+        return cast(
+            "npt.NDArray[np.float32]",
+            text_features.cpu().numpy().astype(np.float32),
+        )

echovector/embeddings/factory.py

@@ -0,0 +1,78 @@
+"""Factory for creating embedding backends."""
+
+from typing import Any, ClassVar
+
+from echovector.embeddings.base import EmbeddingBackend
+
+# Map backend names to their (module, class) for lazy import.
+_BACKEND_LOCATIONS: dict[str, tuple[str, str]] = {
+    "clap": ("echovector.embeddings.clap", "ClapBackend"),
+    "whisper": ("echovector.embeddings.whisper_enc", "WhisperBackend"),
+    "wav2vec2": ("echovector.embeddings.wav2vec2", "Wav2Vec2Backend"),
+    "hubert": ("echovector.embeddings.hubert", "HubertBackend"),
+    "ast": ("echovector.embeddings.ast_model", "ASTBackend"),
+    "local": ("echovector.embeddings.local", "LocalFeatureBackend"),
+    "smoke": ("echovector.embeddings.local", "LocalFeatureBackend"),
+}
+
+
+class EmbeddingFactory:
+    """Factory to instantiate embedding backends by name."""
+
+    _registry: ClassVar[dict[str, type[EmbeddingBackend]]] = {}
+
+    @classmethod
+    def register_backend(cls, name: str, backend_cls: type[EmbeddingBackend]) -> None:
+        """Register a new backend class.
+
+        Args:
+            name: The name to register the backend under.
+            backend_cls: The backend class.
+        """
+        cls._registry[name] = backend_cls
+
+    @classmethod
+    def create(cls, name: str, **kwargs: Any) -> EmbeddingBackend:
+        """Create an embedding backend instance.
+
+        Args:
+            name: The name of the backend (e.g., 'clap', 'whisper').
+            **kwargs: Additional keyword arguments to pass to the backend constructor.
+
+        Returns:
+            An instance of the requested EmbeddingBackend.
+
+        Raises:
+            ValueError: If the backend name is not registered.
+        """
+        import importlib
+
+        name_lower = name.lower()
+
+        if name_lower not in cls._registry:
+            if name_lower not in _BACKEND_LOCATIONS:
+                valid_names = ", ".join(_BACKEND_LOCATIONS.keys())
+                raise ValueError(
+                    f"Unknown embedding backend: '{name}'. Valid options are: {valid_names}"
+                )
+            module_path, class_name = _BACKEND_LOCATIONS[name_lower]
+            module = importlib.import_module(module_path)
+            cls._registry[name_lower] = getattr(module, class_name)
+
+        return cls._registry[name_lower](**kwargs)
+
+
+def get_embedding_model(
+    model_type: str = "clap",
+    **kwargs: Any,
+) -> EmbeddingBackend:
+    """Convenience function to get an embedding model.
+
+    Args:
+        model_type: The type of model to instantiate.
+        **kwargs: Arguments to pass to the model constructor.
+
+    Returns:
+        An instantiated EmbeddingBackend.
+    """
+    return EmbeddingFactory.create(model_type, **kwargs)

echovector/embeddings/hubert.py

@@ -0,0 +1,41 @@
+"""HuBERT embedding backend stub."""
+
+import numpy as np
+import numpy.typing as npt
+
+from echovector.embeddings.base import EmbeddingBackend
+
+
+class HubertBackend(EmbeddingBackend):
+    """Stub implementation for the HuBERT embedding backend."""
+
+    def __init__(self, model_name: str = "facebook/hubert-base-ls960") -> None:
+        """Initialize the stub HuBERT backend."""
+        self.model_name = model_name
+
+    @property
+    def embedding_dim(self) -> int:
+        """Return a stub embedding dimension."""
+        return 768
+
+    def embed_audio(self, audio_paths: list[str]) -> npt.NDArray[np.float32]:
+        """Embed a batch of audio files.
+
+        Args:
+            audio_paths: List of file paths to audio files.
+
+        Raises:
+            NotImplementedError: As this is a stub.
+        """
+        raise NotImplementedError("HuBERT audio embedding not implemented.")
+
+    def embed_text(self, texts: list[str]) -> npt.NDArray[np.float32]:
+        """Embed a batch of text queries.
+
+        Args:
+            texts: List of text strings.
+
+        Raises:
+            NotImplementedError: HuBERT backend does not support text embeddings.
+        """
+        raise NotImplementedError("HuBERT backend does not support text embeddings.")

echovector/embeddings/local.py

@@ -0,0 +1,109 @@
+"""Deterministic local embedding backend for offline smoke tests."""
+
+from typing import cast
+
+import librosa
+import numpy as np
+import numpy.typing as npt
+import soundfile as sf
+
+from echovector.embeddings.base import EmbeddingBackend
+
+
+class LocalFeatureBackend(EmbeddingBackend):
+    """Small dependency-only backend that avoids model downloads.
+
+    This backend is intended for CI, Kaggle smoke tests, and demos. It is not a
+    replacement for CLAP when semantic text/audio alignment matters.
+    """
+
+    def __init__(self, sample_rate: int = 16_000) -> None:
+        """Initialize the backend."""
+        self.sample_rate = sample_rate
+
+    @property
+    def embedding_dim(self) -> int:
+        """Return the feature vector size."""
+        return 8
+
+    def embed_audio(self, audio_paths: list[str]) -> npt.NDArray[np.float32]:
+        """Embed audio files using simple acoustic descriptors."""
+        return np.vstack([self._embed_audio_file(path) for path in audio_paths]).astype(np.float32)
+
+    def embed_text(self, texts: list[str]) -> npt.NDArray[np.float32]:
+        """Embed text queries with a keyword-weighted acoustic proxy."""
+        return np.vstack([self._embed_text(text) for text in texts]).astype(np.float32)
+
+    def _embed_audio_file(self, path: str) -> npt.NDArray[np.float32]:
+        audio, sample_rate = sf.read(path, dtype="float32", always_2d=False)
+        if audio.ndim > 1:
+            audio = np.mean(audio, axis=1)
+        if sample_rate != self.sample_rate:
+            audio = librosa.resample(
+                y=audio,
+                orig_sr=int(sample_rate),
+                target_sr=self.sample_rate,
+                axis=0,
+            )
+
+        if len(audio) == 0:
+            return self._normalize(np.zeros(self.embedding_dim, dtype=np.float32))
+
+        duration = len(audio) / self.sample_rate
+        rms = float(np.sqrt(np.mean(np.square(audio))))
+        zcr = float(np.mean(librosa.feature.zero_crossing_rate(y=audio)[0]))
+        centroid = float(
+            np.mean(librosa.feature.spectral_centroid(y=audio, sr=self.sample_rate)[0])
+            / (self.sample_rate / 2)
+        )
+        bandwidth = float(
+            np.mean(librosa.feature.spectral_bandwidth(y=audio, sr=self.sample_rate)[0])
+            / (self.sample_rate / 2)
+        )
+        flatness = float(np.mean(librosa.feature.spectral_flatness(y=audio)[0]))
+
+        features = np.array(
+            [
+                min(duration / 30.0, 1.0),
+                min(rms * 4.0, 1.0),
+                min(zcr * 8.0, 1.0),
+                min(centroid, 1.0),
+                min(bandwidth, 1.0),
+                min(flatness * 10.0, 1.0),
+                1.0 if rms < 0.01 else 0.0,
+                max(1.0 - min(centroid, 1.0), 0.0),
+            ],
+            dtype=np.float32,
+        )
+        return self._normalize(features)
+
+    def _embed_text(self, text: str) -> npt.NDArray[np.float32]:
+        lowered = text.lower()
+        features = np.full(self.embedding_dim, 0.05, dtype=np.float32)
+
+        if any(term in lowered for term in ("long", "duration", "extended")):
+            features[0] = 1.0
+        if any(term in lowered for term in ("loud", "strong", "alarm", "alert")):
+            features[1] = 1.0
+        if any(term in lowered for term in ("noisy", "buzz", "speech", "busy")):
+            features[2] = 1.0
+        if any(term in lowered for term in ("high", "treble", "bright", "alarm", "alert")):
+            features[3] = 1.0
+        if any(term in lowered for term in ("wide", "broadband", "noise")):
+            features[4] = 1.0
+        if any(term in lowered for term in ("flat", "noise", "static")):
+            features[5] = 1.0
+        if any(term in lowered for term in ("silent", "silence", "quiet")):
+            features[6] = 1.0
+        if any(term in lowered for term in ("low", "bass", "deep")):
+            features[3] = 0.0
+            features[4] = max(features[4], 0.2)
+            features[7] = 1.0
+
+        return self._normalize(features)
+
+    def _normalize(self, vector: npt.NDArray[np.float32]) -> npt.NDArray[np.float32]:
+        norm = float(np.linalg.norm(vector))
+        if norm == 0.0:
+            return vector
+        return cast("npt.NDArray[np.float32]", (vector / norm).astype(np.float32))

echovector/embeddings/wav2vec2.py

@@ -0,0 +1,41 @@
+"""Wav2Vec2 embedding backend stub."""
+
+import numpy as np
+import numpy.typing as npt
+
+from echovector.embeddings.base import EmbeddingBackend
+
+
+class Wav2Vec2Backend(EmbeddingBackend):
+    """Stub implementation for the Wav2Vec2 embedding backend."""
+
+    def __init__(self, model_name: str = "facebook/wav2vec2-base") -> None:
+        """Initialize the stub Wav2Vec2 backend."""
+        self.model_name = model_name
+
+    @property
+    def embedding_dim(self) -> int:
+        """Return a stub embedding dimension."""
+        return 768
+
+    def embed_audio(self, audio_paths: list[str]) -> npt.NDArray[np.float32]:
+        """Embed a batch of audio files.
+
+        Args:
+            audio_paths: List of file paths to audio files.
+
+        Raises:
+            NotImplementedError: As this is a stub.
+        """
+        raise NotImplementedError("Wav2Vec2 audio embedding not implemented.")
+
+    def embed_text(self, texts: list[str]) -> npt.NDArray[np.float32]:
+        """Embed a batch of text queries.
+
+        Args:
+            texts: List of text strings.
+
+        Raises:
+            NotImplementedError: Wav2Vec2 backend does not support text embeddings.
+        """
+        raise NotImplementedError("Wav2Vec2 backend does not support text embeddings.")

echovector/embeddings/whisper_enc.py

@@ -0,0 +1,44 @@
+"""Whisper encoder embedding backend stub."""
+
+import numpy as np
+import numpy.typing as npt
+
+from echovector.embeddings.base import EmbeddingBackend
+
+
+class WhisperBackend(EmbeddingBackend):
+    """Stub implementation for the Whisper embedding backend.
+
+    Uses the encoder of Whisper models to extract audio features.
+    """
+
+    def __init__(self, model_name: str = "openai/whisper-base") -> None:
+        """Initialize the stub Whisper backend."""
+        self.model_name = model_name
+
+    @property
+    def embedding_dim(self) -> int:
+        """Return a stub embedding dimension."""
+        return 512
+
+    def embed_audio(self, audio_paths: list[str]) -> npt.NDArray[np.float32]:
+        """Embed a batch of audio files.
+
+        Args:
+            audio_paths: List of file paths to audio files.
+
+        Raises:
+            NotImplementedError: As this is a stub.
+        """
+        raise NotImplementedError("Whisper audio embedding not implemented.")
+
+    def embed_text(self, texts: list[str]) -> npt.NDArray[np.float32]:
+        """Embed a batch of text queries.
+
+        Args:
+            texts: List of text strings.
+
+        Raises:
+            NotImplementedError: Whisper backend does not support text embeddings.
+        """
+        raise NotImplementedError("Whisper backend does not support text embeddings.")

echovector/evaluation/__init__.py

@@ -0,0 +1 @@
+"""Evaluation module initialization."""

echovector/evaluation/metrics.py

@@ -0,0 +1,45 @@
+"""Evaluation metrics for comparing audio vectors and embeddings."""
+
+import numpy as np
+import numpy.typing as npt
+
+
+def cosine_similarity(vec1: npt.NDArray[np.float32], vec2: npt.NDArray[np.float32]) -> float:
+    """Calculate the cosine similarity between two vectors.
+
+    Args:
+        vec1: First vector.
+        vec2: Second vector.
+
+    Returns:
+        Cosine similarity score between -1.0 and 1.0.
+    """
+    if vec1.shape != vec2.shape:
+        raise ValueError("Vectors must have the same shape.")
+
+    norm1 = np.linalg.norm(vec1)
+    norm2 = np.linalg.norm(vec2)
+
+    if norm1 == 0 or norm2 == 0:
+        return 0.0
+
+    dot_product = np.dot(vec1.flatten(), vec2.flatten())
+    similarity = dot_product / (norm1 * norm2)
+    return float(similarity)
+
+
+def euclidean_distance(vec1: npt.NDArray[np.float32], vec2: npt.NDArray[np.float32]) -> float:
+    """Calculate the Euclidean distance between two vectors.
+
+    Args:
+        vec1: First vector.
+        vec2: Second vector.
+
+    Returns:
+        Euclidean distance.
+    """
+    if vec1.shape != vec2.shape:
+        raise ValueError("Vectors must have the same shape.")
+
+    distance = np.linalg.norm(vec1.flatten() - vec2.flatten())
+    return float(distance)

echovector/indexing/__init__.py

@@ -0,0 +1,12 @@
+"""EchoVector indexing module."""
+
+from .base import BaseIndex, BaseStore
+from .faiss_index import FaissIndex
+from .store import SQLiteStore
+
+__all__ = [
+    "BaseIndex",
+    "BaseStore",
+    "FaissIndex",
+    "SQLiteStore",
+]

echovector/indexing/base.py

@@ -0,0 +1,105 @@
+"""Base interfaces for the indexing module."""
+
+from abc import ABC, abstractmethod
+from typing import Any
+
+import numpy as np
+import numpy.typing as npt
+
+
+class BaseStore(ABC):
+    """Abstract base class for metadata storage."""
+
+    @abstractmethod
+    def initialize(self) -> None:
+        """Initialize the store (e.g., create tables)."""
+
+    @abstractmethod
+    def add(
+        self, int_ids: list[int], string_ids: list[str], metadata_list: list[dict[str, Any]]
+    ) -> None:
+        """Add metadata and ID mappings to the store.
+
+        Args:
+            int_ids: List of integer IDs assigned by the index.
+            string_ids: List of original string IDs.
+            metadata_list: List of metadata dictionaries.
+        """
+
+    @abstractmethod
+    def get_by_int_ids(
+        self, int_ids: list[int]
+    ) -> tuple[list[str | None], list[dict[str, Any] | None]]:
+        """Retrieve string IDs and metadata for a list of integer IDs.
+
+        Args:
+            int_ids: List of integer IDs to query.
+
+        Returns:
+            A tuple containing a list of string IDs and a list of metadata dictionaries.
+        """
+
+    @abstractmethod
+    def get_max_int_id(self) -> int:
+        """Get the maximum integer ID currently in the store.
+
+        Returns:
+            The maximum integer ID, or -1 if empty.
+        """
+
+    @abstractmethod
+    def close(self) -> None:
+        """Close any open connections."""
+
+
+class BaseIndex(ABC):
+    """Abstract base class for vector indices."""
+
+    @abstractmethod
+    def add(
+        self,
+        embeddings: npt.NDArray[np.float32],
+        ids: list[str],
+        metadata: list[dict[str, Any]] | None = None,
+    ) -> None:
+        """Add embeddings, IDs, and metadata to the index.
+
+        Args:
+            embeddings: A 2D numpy array of embeddings.
+            ids: A list of string IDs corresponding to the embeddings.
+            metadata: An optional list of metadata dictionaries.
+        """
+
+    @abstractmethod
+    def search(
+        self, query_embeddings: npt.NDArray[np.float32], k: int = 10
+    ) -> tuple[
+        npt.NDArray[np.float32],
+        list[list[str | None]],
+        list[list[dict[str, Any] | None]],
+    ]:
+        """Search for the k nearest neighbors for each query.
+
+        Args:
+            query_embeddings: A 2D numpy array of query embeddings.
+            k: The number of nearest neighbors to retrieve.
+
+        Returns:
+            A tuple of (distances, string_ids, metadata).
+        """
+
+    @abstractmethod
+    def save(self, index_path: str) -> None:
+        """Save the index to disk.
+
+        Args:
+            index_path: The file path to save the index to.
+        """
+
+    @abstractmethod
+    def load(self, index_path: str) -> None:
+        """Load the index from disk.
+
+        Args:
+            index_path: The file path to load the index from.
+        """