rvcbench 2.0.0__py3-none-any.whl

rvcbench/__init__.py

@@ -0,0 +1,17 @@
+"""RVCBench: a benchmark for voice cloning robustness and audio protection."""
+from importlib.metadata import PackageNotFoundError, version
+
+try:
+    __version__ = version("rvcbench")
+except PackageNotFoundError:  # source checkout that has not been installed
+    __version__ = "0+unknown"
+
+__all__ = ["VoiceCloningAdapter", "__version__"]
+
+
+def __getattr__(name):
+    # Imported on demand so that `import rvcbench` stays free of heavy dependencies.
+    if name == "VoiceCloningAdapter":
+        from rvcbench.adapter import VoiceCloningAdapter
+        return VoiceCloningAdapter
+    raise AttributeError(f"module 'rvcbench' has no attribute {name!r}")

rvcbench/adapter.py

@@ -0,0 +1,98 @@
+"""Public base class for evaluating your own voice cloning model with RVCBench.
+
+Subclass :class:`VoiceCloningAdapter`, implement :meth:`clone`, and select the class
+with ``vc.adapter=your_package.your_module:YourAdapter``. The benchmark runner handles
+sample selection, seeding, output naming, run records and scoring.
+"""
+from __future__ import annotations
+
+import time
+from pathlib import Path
+from typing import Optional, Tuple
+
+import numpy as np
+import soundfile as sf
+
+from rvcbench.adversary.base_adversary import BaseAdversary
+
+
+class VoiceCloningAdapter(BaseAdversary):
+    """Zero-shot voice cloning model driven one utterance at a time.
+
+    ``self.config`` is the ``adversary`` block of the run config, ``self.device`` the
+    requested device and ``self.logger`` a standard logger.
+    """
+
+    #: Label recorded with each synthesis time. Describe what :meth:`clone` measures.
+    timing_scope = "external_adapter_clone_call_excluding_output_write"
+
+    def __init__(self, config, dataset_config, device, logger):
+        super().__init__(config, device)
+        self.dataset_config = dataset_config
+        self.logger = logger
+        self._loaded = False
+
+    def load(self) -> None:
+        """Load weights and other resources once per run. Optional."""
+
+    def clone(self, *, text: str, reference_audio: Path, reference_text: str,
+              language: Optional[str]) -> Tuple[np.ndarray, int]:
+        """Speak ``text`` in the voice of ``reference_audio``.
+
+        Return ``(waveform, sample_rate)``: a mono float waveform in ``[-1, 1]`` and its
+        sample rate in Hz. Raise an exception when the utterance cannot be generated;
+        the runner records the failure for that sample and continues.
+        """
+        raise NotImplementedError
+
+    def unload(self) -> None:
+        """Release resources at the end of the run. Optional."""
+
+    # -- runner integration -------------------------------------------------------
+
+    def _ensure_model(self) -> None:
+        if not self._loaded:
+            self.load()
+            self._loaded = True
+
+    def close(self) -> None:
+        try:
+            if self._loaded:
+                self.unload()
+        finally:
+            self._loaded = False
+            super().close()
+
+    def attack(self, *, output_path, dataset, protected_audio_path=None):
+        del protected_audio_path  # protected references already replace sample.prompt_path
+        self._ensure_model()
+        output_dir = Path(output_path).resolve()
+        output_dir.mkdir(parents=True, exist_ok=True)
+        self._init_synthesis_timings(output_dir)
+        try:
+            for sample in dataset.get_zero_shot_samples(max_samples=self.config.get("max_samples")):
+                reference = self._resolve_prompt_path(sample)
+                if reference is None:
+                    raise FileNotFoundError(f"Missing reference audio: {sample.prompt_path}")
+                text = (sample.target_text or "").strip()
+                if not text:
+                    raise ValueError("Target text is empty")
+                started = time.perf_counter()
+                waveform, sample_rate = self.clone(
+                    text=text, reference_audio=reference,
+                    reference_text=(sample.prompt_text or "").strip(),
+                    language=sample.target_language or sample.prompt_language)
+                elapsed = time.perf_counter() - started
+                waveform = np.asarray(waveform, dtype=np.float32)
+                if waveform.ndim == 2 and 1 in waveform.shape:
+                    waveform = waveform.reshape(-1)
+                if waveform.ndim != 1 or not waveform.size or not np.isfinite(waveform).all():
+                    raise ValueError("clone() must return nonempty finite mono audio")
+                if int(sample_rate) <= 0:
+                    raise ValueError("clone() must return a positive sample rate")
+                path = (self._speaker_output_dir(output_dir, str(sample.speaker_id))
+                        / self._cloned_filename(sample, sample.index))
+                sf.write(str(path), np.clip(waveform, -1.0, 1.0), int(sample_rate))
+                self._record_synthesis_timing(path, elapsed)
+        finally:
+            self._flush_synthesis_timings()

rvcbench/adversary/bark_voice_clone_ots.py

@@ -0,0 +1,146 @@
+from __future__ import annotations
+
+from pathlib import Path
+import time
+from typing import TYPE_CHECKING, Optional
+
+import numpy as np
+import soundfile as sf
+from hydra.utils import to_absolute_path
+
+from .base_adversary import BaseAdversary
+
+if TYPE_CHECKING:
+    from rvcbench.models.bark_voice_clone import BarkVoiceCloneGenerator
+
+
+class BarkVoiceCloneZeroShotAdversary(BaseAdversary):
+    """Runs the Bark voice cloning pipeline for zero-shot attacks."""
+
+    def __init__(self, config, dataset_config, device, logger):
+        super().__init__(config, device)
+        self.dataset_config = dataset_config
+        self.logger = logger
+
+        self.code_path = Path(to_absolute_path(self.config.code_path)).resolve()
+        self.models_dir = self._resolve_optional_path(self.config.get("models_dir"))
+        self.cache_dir = self._resolve_optional_path(self.config.get("cache_dir"))
+        self.prompt_cache_dir = self._resolve_optional_path(self.config.get("prompt_cache_dir"))
+        self.hubert_checkpoint = self._resolve_optional_path(self.config.get("hubert_checkpoint"))
+        self.hubert_tokenizer = self._resolve_optional_path(self.config.get("hubert_tokenizer"))
+
+        self.reference_assignment = str(
+            self.config.get("reference_assignment", "round_robin")
+        ).lower().strip()
+        self.max_samples = self.config.get("max_samples")
+        self.seed = self.config.get("seed")
+        self.default_prompt_text = str(
+            self.config.get(
+                "default_prompt_text",
+                "Here is a short sample of the desired voice.",
+            )
+        )
+
+        self._generator: Optional[BarkVoiceCloneGenerator] = None
+
+    # ------------------------------------------------------------------
+    # Helpers
+    # ------------------------------------------------------------------
+    def _resolve_optional_path(self, value) -> Optional[Path]:
+        if value in (None, ""):
+            return None
+        resolved = Path(to_absolute_path(str(value)))
+        return resolved.resolve()
+
+    def _coerce_optional(self, key: str, caster):
+        value = self.config.get(key)
+        if value in (None, ""):
+            return None
+        try:
+            return caster(value)
+        except (TypeError, ValueError):
+            if self.logger is not None:
+                self.logger.warning(
+                    "[BarkVC] Failed to cast '%s' value '%s'; ignoring.",
+                    key,
+                    value,
+                )
+            return None
+
+    def _ensure_generator(self) -> None:
+        if self._generator is not None:
+            return
+        from rvcbench.models.bark_voice_clone import BarkVoiceCloneGenerator, BarkVoiceCloneGeneratorConfig
+
+        generator_config = BarkVoiceCloneGeneratorConfig(
+            code_path=self.code_path,
+            models_dir=self.models_dir,
+            cache_dir=self.cache_dir,
+            prompt_cache_dir=self.prompt_cache_dir,
+            hubert_checkpoint=self.hubert_checkpoint,
+            hubert_tokenizer=self.hubert_tokenizer,
+            text_tokenizer_path=self._resolve_optional_path(self.config.get('text_tokenizer_path')),
+            text_temperature=float(self.config.get("text_temperature", 0.7)),
+            text_top_k=self._coerce_optional("text_top_k", int),
+            text_top_p=self._coerce_optional("text_top_p", float),
+            coarse_temperature=float(self.config.get("coarse_temperature", 0.7)),
+            coarse_top_k=self._coerce_optional("coarse_top_k", int),
+            coarse_top_p=self._coerce_optional("coarse_top_p", float),
+            fine_temperature=float(self.config.get("fine_temperature", 0.5)),
+            semantic_use_kv_cache=bool(self.config.get("semantic_use_kv_cache", True)),
+            coarse_use_kv_cache=bool(self.config.get("coarse_use_kv_cache", True)),
+            silent=bool(self.config.get("silent", True)),
+            force_reload_models=bool(self.config.get("force_reload_models", False)),
+            max_prompt_seconds=self._coerce_optional("max_prompt_seconds", float),
+            hubert_layer=int(self.config.get('hubert_layer', 9)),
+        )
+        self._generator = BarkVoiceCloneGenerator(generator_config, self.device, self.logger)
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+    def generate_sample(self, sample, *, output_dir):
+        reference_path = self._resolve_prompt_path(sample)
+        if reference_path is None:
+            raise FileNotFoundError(f"Missing Bark reference audio: {sample.prompt_path}")
+        text = (sample.target_text or "").strip()
+        if not text:
+            raise ValueError('Bark requires nonempty target text')
+        self._ensure_generator()
+        self._generator.ensure_model()
+        seed = self._sample_seed(sample)
+        if seed is not None:
+            from rvcbench.utils.seeding import configure_seeds
+            configure_seeds(seed, logger=None)
+        self._log_clone_request("BarkVC", 0, 1, str(sample.speaker_id), reference_path,
+                                text, prompt_transcript=sample.prompt_text or self.default_prompt_text)
+        started = time.perf_counter()
+        audio, sample_rate = self._generator.generate(
+            text=text, prompt_audio=reference_path, sample_index=sample.index)
+        elapsed = time.perf_counter() - started
+        audio = np.asarray(audio, dtype=np.float32)
+        if audio.ndim != 1 or not audio.size or not np.isfinite(audio).all():
+            raise ValueError('Bark returned invalid mono audio')
+        path = self._speaker_output_dir(Path(output_dir).resolve(), str(sample.speaker_id)) / self._cloned_filename(sample, sample.index)
+        sf.write(path, audio, sample_rate)
+        return path, elapsed
+
+    def attack(self, *, output_path, dataset, protected_audio_path=None):
+        del protected_audio_path
+        output_dir = Path(output_path).resolve()
+        output_dir.mkdir(parents=True, exist_ok=True)
+        self._init_synthesis_timings(output_dir)
+        samples = dataset.get_zero_shot_samples(max_samples=self.max_samples)
+        if not samples:
+            raise RuntimeError("No zero-shot samples available for Bark voice cloning adversary.")
+        self._log_attack_plan("BarkVC", samples, self._count_available_prompts(samples))
+        completed = 0
+        try:
+            for sample in samples:
+                path, elapsed = self.generate_sample(sample, output_dir=output_dir)
+                self._record_synthesis_timing(path, elapsed)
+                completed += 1
+        finally:
+            self._flush_synthesis_timings()
+        if self.logger:
+            self.logger.info("[BarkVC] Generated %d/%d utterances.", completed, len(samples))

rvcbench/adversary/base_adversary.py

@@ -0,0 +1,205 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+import csv
+import re
+from pathlib import Path
+from typing import Optional, Sequence, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from rvcbench.datasets.data_utils import AllSpeakerData
+    from rvcbench.datasets.data_utils import ZeroShotSample
+
+class BaseAdversary(ABC):
+    """Abstract Base Class for all malicious adversaries."""
+    def __init__(self, config, device):
+        # Hydra sometimes provides the full run config; extract the adversary block if present.
+        adversary_cfg = getattr(config, "adversary", None)
+        self.config = adversary_cfg or config
+        self.device = device
+        self._synthesis_timing_records = []
+        self._synthesis_timing_path: Optional[Path] = None
+        self._managed_lifetime = False
+
+    def _sample_seed(self, sample):
+        seed = self.config.get('seed')
+        return None if seed is None else int(seed) + int(sample.index)
+
+    def prepare(self):
+        """Load adapter resources once so setup failures are run-level failures."""
+        self._managed_lifetime = True
+        for name in ('_ensure_generator', '_ensure_synthesizer', '_ensure_model', '_ensure_imports'):
+            hook = getattr(self, name, None)
+            if hook is not None:
+                hook()
+                break
+        for name in ('_generator', '_synthesizer'):
+            resource = getattr(self, name, None)
+            ensure = getattr(resource, 'ensure_model', None)
+            if ensure is not None:
+                ensure()
+
+    def close(self) -> None:
+        """Release owned generators, including persistent subprocess workers."""
+        errors = []
+        for name in ('_generator', '_synthesizer'):
+            resource = getattr(self, name, None)
+            if resource is None:
+                continue
+            try:
+                close = getattr(resource, 'close', None)
+                if close is not None:
+                    close()
+            except Exception as exc:
+                errors.append(exc)
+            else:
+                setattr(self, name, None)
+        if errors:
+            raise RuntimeError('Failed to close adapter resources') from errors[0]
+        self._managed_lifetime = False
+
+    def _speaker_slug(self, speaker_id: str) -> str:
+        """Return a filesystem-safe identifier for the supplied speaker identifier."""
+        token = str(speaker_id or "").strip()
+        slug = re.sub(r"[^0-9A-Za-z_.-]", "_", token)
+        return slug or "unknown"
+
+    def _speaker_output_dir(self, base_dir: Path, speaker_id: str) -> Path:
+        """Ensure the per-speaker directory exists and return its path."""
+        speaker_dir = Path(base_dir) / self._speaker_slug(speaker_id)
+        speaker_dir.mkdir(parents=True, exist_ok=True)
+        return speaker_dir
+
+    def _resolve_prompt_path(self, sample) -> Optional[Path]:
+        raw_path = getattr(sample, "prompt_path", None)
+        if raw_path in (None, ""):
+            return None
+        path = Path(str(raw_path))
+        try:
+            candidate = path if path.is_absolute() else path.resolve(strict=False)
+        except Exception:
+            candidate = path
+        if candidate.exists():
+            return candidate
+        return None
+
+    def _count_available_prompts(self, samples: Sequence["ZeroShotSample"]) -> int:
+        return sum(1 for sample in samples if self._resolve_prompt_path(sample) is not None)
+
+    def _preview_text(self, text: Optional[str], limit: int = 120) -> str:
+        snippet = re.sub(r"\s+", " ", (text or "").strip())
+        if len(snippet) > limit:
+            return snippet[: limit - 1] + "…"
+        return snippet or "<empty>"
+
+    def _describe_audio(self, path: Optional[Path]) -> str:
+        if path is None:
+            return "<none>"
+        try:
+            import soundfile as sf  # Local import to avoid hard dependency when unused
+
+            with sf.SoundFile(str(path)) as handle:
+                duration = handle.frames / handle.samplerate if handle.samplerate else 0.0
+                return f"{path.name} (sr={handle.samplerate}, dur={duration:.2f}s)"
+        except Exception:
+            return path.name
+
+    def _log_attack_plan(
+        self,
+        model_label: str,
+        samples: Sequence["ZeroShotSample"],
+        prompt_count: int,
+    ) -> None:
+        logger = getattr(self, "logger", None)
+        if not logger:
+            return
+        total = len(samples)
+        speakers = [str(sample.speaker_id) for sample in samples]
+        unique_speakers = sorted(set(speakers))
+        logger.info(
+            "[%s] Preparing %d utterances across %d speakers. Prompt audios available: %d.",
+            model_label,
+            total,
+            len(unique_speakers),
+            prompt_count,
+        )
+        if unique_speakers:
+            preview = ", ".join(unique_speakers[:8])
+            if len(unique_speakers) > 8:
+                preview += ", …"
+            logger.debug("[%s] Speaker roster: %s", model_label, preview)
+
+    def _log_clone_request(
+        self,
+        model_label: str,
+        index: int,
+        total: int,
+        speaker_id: str,
+        prompt_path: Optional[Path],
+        spoken_text: Optional[str],
+        *,
+        prompt_transcript: Optional[str] = None,
+    ) -> None:
+        logger = getattr(self, "logger", None)
+        if not logger:
+            return
+        prompt_desc = self._describe_audio(prompt_path)
+        text_preview = self._preview_text(spoken_text)
+        message = (
+            f"[{model_label}] [{index + 1}/{total}] speaker={speaker_id} "
+            f"prompt={prompt_desc} text=\"{text_preview}\""
+        )
+        if prompt_transcript:
+            message += f" transcript=\"{self._preview_text(prompt_transcript)}\""
+        logger.info(message)
+
+    def _cloned_filename(self, sample, idx: int, suffix: str = "cloned") -> str:
+        from rvcbench.benchmark.artifacts import output_path
+        return output_path(Path('.'), sample, suffix).name
+
+    def _init_synthesis_timings(self, output_dir: Path) -> None:
+        self._synthesis_timing_records = []
+        self._synthesis_timing_path = Path(output_dir) / "synthesis_timings.csv"
+
+    def _record_synthesis_timing(self, generated_path: Path, elapsed_sec: Optional[float]) -> None:
+        if elapsed_sec is None:
+            return
+        if self._synthesis_timing_path is None:
+            return
+        try:
+            resolved = generated_path.resolve()
+        except Exception:
+            resolved = generated_path
+        self._synthesis_timing_records.append(
+            {
+                "generated_path": str(resolved),
+                "synthesis_time_sec": float(elapsed_sec),
+                "timing_scope": getattr(self, "timing_scope", "adapter_reported_synthesis"),
+            }
+        )
+
+    def _flush_synthesis_timings(self) -> None:
+        if not self._synthesis_timing_path or not self._synthesis_timing_records:
+            return
+        self._synthesis_timing_path.parent.mkdir(parents=True, exist_ok=True)
+        with open(self._synthesis_timing_path, "w", encoding="utf-8", newline="") as csvfile:
+            writer = csv.DictWriter(
+                csvfile,
+                fieldnames=["generated_path", "synthesis_time_sec", "timing_scope"],
+            )
+            writer.writeheader()
+            writer.writerows(self._synthesis_timing_records)
+        logger = getattr(self, "logger", None)
+        if logger is not None:
+            logger.info("Saved per-sample synthesis timings to %s", self._synthesis_timing_path)
+
+    @abstractmethod
+    def attack(
+        self,
+        *,
+        output_path: str,
+        dataset: "AllSpeakerData",
+        protected_audio_path: Optional[str] = None,
+    ) -> None:
+        """Generate adversarial audio for the provided dataset."""
+        pass