inaVAMOS 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
inavamos/__init__.py ADDED
@@ -0,0 +1,9 @@
1
+ """inaVAMOS: Voice And Music Open Segmenter. Speech (voice activity) and music detection based on SSL models from INA."""
2
+
3
+ from .audio import load_audio
4
+ from .models import DETECTORS, ModelAccessError
5
+ from .segmenter import Result, Segment, Segmenter
6
+
7
+ __version__ = "0.1.0"
8
+
9
+ __all__ = ["Segmenter", "Segment", "Result", "load_audio", "DETECTORS", "ModelAccessError", "__version__"]
inavamos/__main__.py ADDED
@@ -0,0 +1,5 @@
1
+ import sys
2
+
3
+ from .cli import main
4
+
5
+ sys.exit(main())
inavamos/audio.py ADDED
@@ -0,0 +1,57 @@
1
+ """Audio decoding to 16 kHz mono float32."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import os
6
+ import shutil
7
+ import subprocess
8
+
9
+ import numpy as np
10
+
11
+ SAMPLING_RATE = 16000
12
+
13
+
14
+ def load_audio(path, sr: int = SAMPLING_RATE, start: float | None = None, stop: float | None = None) -> np.ndarray:
15
+ """Decode any audio or video file to a mono float32 signal at ``sr`` Hz.
16
+
17
+ ffmpeg is used when available (it handles virtually every media format);
18
+ otherwise the file is decoded with librosa/soundfile.
19
+
20
+ Args:
21
+ path: media file path (or URL, when ffmpeg is used).
22
+ sr: target sampling rate.
23
+ start: optional start time, in seconds.
24
+ stop: optional stop time, in seconds.
25
+ """
26
+ if start is not None and start < 0:
27
+ raise ValueError("start must be positive")
28
+ if start is not None and stop is not None and stop <= start:
29
+ raise ValueError("stop must be greater than start")
30
+ if "://" not in str(path) and not os.path.exists(path):
31
+ raise FileNotFoundError(f"No such file: {path}")
32
+
33
+ if shutil.which("ffmpeg"):
34
+ return _load_ffmpeg(path, sr, start, stop)
35
+ return _load_librosa(path, sr, start, stop)
36
+
37
+
38
+ def _load_ffmpeg(path, sr, start, stop):
39
+ cmd = ["ffmpeg", "-nostdin", "-hide_banner", "-loglevel", "error"]
40
+ if start:
41
+ cmd += ["-ss", str(start)]
42
+ cmd += ["-i", str(path)]
43
+ if stop is not None:
44
+ cmd += ["-t", str(stop - (start or 0))]
45
+ cmd += ["-map", "0:a:0", "-vn", "-ac", "1", "-ar", str(sr), "-f", "f32le", "-acodec", "pcm_f32le", "-"]
46
+ proc = subprocess.run(cmd, capture_output=True)
47
+ if proc.returncode != 0:
48
+ raise RuntimeError(f"ffmpeg failed to decode {path}:\n{proc.stderr.decode(errors='replace').strip()}")
49
+ return np.frombuffer(proc.stdout, dtype=np.float32).copy()
50
+
51
+
52
+ def _load_librosa(path, sr, start, stop):
53
+ import librosa
54
+
55
+ duration = None if stop is None else stop - (start or 0)
56
+ audio, _ = librosa.load(path, sr=sr, mono=True, offset=start or 0.0, duration=duration)
57
+ return audio.astype(np.float32)
inavamos/cli.py ADDED
@@ -0,0 +1,132 @@
1
+ """Command line interface: ``ina-vamos -i media.mp3 -o output_dir``."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import argparse
6
+ import logging
7
+ import sys
8
+ import time
9
+ from pathlib import Path
10
+
11
+ from . import __version__
12
+ from .export import FORMATS, export
13
+ from .models import DETECTORS, ModelAccessError
14
+ from .segmenter import Segmenter
15
+
16
+ logger = logging.getLogger("inavamos")
17
+
18
+
19
+ def build_parser() -> argparse.ArgumentParser:
20
+ parser = argparse.ArgumentParser(
21
+ prog="ina-vamos",
22
+ description=(
23
+ "inaVAMOS (Voice And Music Open Segmenter): speech (voice activity) and music "
24
+ "detection in audio and video files, using SSL-based models from INA (https://huggingface.co/ina-foss)."
25
+ ),
26
+ )
27
+ parser.add_argument("-i", "--input", nargs="+", required=True, help="media files to process (any format read by ffmpeg)")
28
+ parser.add_argument(
29
+ "-o", "--output-dir", type=Path, help="directory where results are written (default: print to standard output)"
30
+ )
31
+ parser.add_argument("-f", "--format", choices=list(FORMATS), default="csv", help="output format (default: csv)")
32
+ parser.add_argument(
33
+ "-d",
34
+ "--detectors",
35
+ nargs="+",
36
+ default=list(DETECTORS),
37
+ metavar="{speech,music}",
38
+ help="detectors to run (default: speech music). 'vad' is an alias of 'speech'",
39
+ )
40
+ parser.add_argument("--device", help="torch device, e.g. cpu, cuda, cuda:1 (default: cuda if available)")
41
+ parser.add_argument("--batch-size", type=int, default=8, help="number of windows processed together (default: 8)")
42
+ parser.add_argument(
43
+ "--window-duration",
44
+ type=float,
45
+ default=30.0,
46
+ help="duration in seconds of the analysis windows (default: 30). 0 processes files at once",
47
+ )
48
+ parser.add_argument(
49
+ "--context-duration",
50
+ type=float,
51
+ default=2.5,
52
+ help="overlap in seconds on each side of the windows (default: 2.5)",
53
+ )
54
+ parser.add_argument("--start", type=float, help="start time in seconds of the portion to process")
55
+ parser.add_argument("--stop", type=float, help="stop time in seconds of the portion to process")
56
+ parser.add_argument("--token", help="HuggingFace token, only needed for private model repositories")
57
+ parser.add_argument("--revision", help="revision of the models on the HuggingFace Hub")
58
+ parser.add_argument("--overwrite", action="store_true", help="overwrite existing output files instead of skipping them")
59
+ parser.add_argument("-v", "--verbose", action="store_true", help="print progress information")
60
+ parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
61
+ return parser
62
+
63
+
64
+ def main(argv=None) -> int:
65
+ args = build_parser().parse_args(argv)
66
+ logging.basicConfig(format="%(levelname)s: %(message)s", level=logging.INFO if args.verbose else logging.WARNING)
67
+ logging.getLogger("httpx").setLevel(logging.WARNING)
68
+
69
+ inputs = [Path(p) for p in args.input]
70
+ outputs = {}
71
+ if args.output_dir:
72
+ args.output_dir.mkdir(parents=True, exist_ok=True)
73
+ for path in inputs:
74
+ output = args.output_dir / (path.stem + FORMATS[args.format])
75
+ if output in outputs.values():
76
+ logger.error("Several inputs would be written to %s", output)
77
+ return 2
78
+ outputs[path] = output
79
+ if not args.overwrite:
80
+ todo = [p for p in inputs if not outputs[p].exists()]
81
+ if len(todo) < len(inputs):
82
+ logger.warning("Skipping %d files already processed (use --overwrite)", len(inputs) - len(todo))
83
+ inputs = todo
84
+ if not inputs:
85
+ return 0
86
+
87
+ try:
88
+ segmenter = Segmenter(
89
+ detectors=args.detectors,
90
+ device=args.device,
91
+ window_duration=args.window_duration or None,
92
+ context_duration=args.context_duration,
93
+ batch_size=args.batch_size,
94
+ token=args.token,
95
+ revision=args.revision,
96
+ )
97
+ except (ModelAccessError, ValueError) as e:
98
+ logger.error("%s", e)
99
+ return 2
100
+
101
+ repo_ids = {name: spec.repo_id for name, spec in segmenter.specs.items()}
102
+ failures = 0
103
+ for i, path in enumerate(inputs, 1):
104
+ t0 = time.perf_counter()
105
+ try:
106
+ result = segmenter.process(path, start=args.start, stop=args.stop)
107
+ except Exception as e: # keep processing the other files
108
+ logger.error("%s: %s", path, e)
109
+ failures += 1
110
+ continue
111
+ elapsed = time.perf_counter() - t0
112
+ logger.info(
113
+ "[%d/%d] %s: %.1fs of audio in %.1fs (%.0fx real time)",
114
+ i,
115
+ len(inputs),
116
+ path,
117
+ result.duration,
118
+ elapsed,
119
+ result.duration / max(elapsed, 1e-6),
120
+ )
121
+ content = export(result, args.format, file=str(path), models=repo_ids)
122
+ if args.output_dir:
123
+ outputs[path].write_text(content)
124
+ else:
125
+ if len(inputs) > 1:
126
+ print(f"# {path}")
127
+ sys.stdout.write(content)
128
+ return 1 if failures else 0
129
+
130
+
131
+ if __name__ == "__main__":
132
+ sys.exit(main())
inavamos/export.py ADDED
@@ -0,0 +1,94 @@
1
+ """Export of segmentation results to common file formats."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import csv
6
+ import io
7
+ import json
8
+
9
+ from .segmenter import Result
10
+
11
+ FORMATS = {
12
+ "csv": ".csv",
13
+ "tsv": ".tsv",
14
+ "json": ".json",
15
+ "textgrid": ".TextGrid",
16
+ "audacity": ".txt",
17
+ }
18
+
19
+
20
+ def to_csv(result: Result, delimiter: str = ",") -> str:
21
+ """Combined timeline as a table with ``label,start,stop`` columns."""
22
+ out = io.StringIO()
23
+ writer = csv.writer(out, delimiter=delimiter, lineterminator="\n")
24
+ writer.writerow(["label", "start", "stop"])
25
+ for label, start, stop in result.timeline():
26
+ writer.writerow([label, f"{start:.3f}", f"{stop:.3f}"])
27
+ return out.getvalue()
28
+
29
+
30
+ def to_json(result: Result, **metadata) -> str:
31
+ """Per-detector segments and combined timeline as JSON."""
32
+ return json.dumps({**metadata, **result.to_dict()}, indent=2)
33
+
34
+
35
+ def to_audacity(result: Result) -> str:
36
+ """Combined timeline as an Audacity label track (File > Import > Labels)."""
37
+ return "".join(f"{start:.6f}\t{stop:.6f}\t{label}\n" for label, start, stop in result.timeline())
38
+
39
+
40
+ def _quote(text: str) -> str:
41
+ return '"' + text.replace('"', '""') + '"'
42
+
43
+
44
+ def to_textgrid(result: Result) -> str:
45
+ """Praat TextGrid with one tier per detector and a tier for the combined timeline."""
46
+ xmin, xmax = result.offset, result.offset + result.duration
47
+ tiers = [(name, result.segments(name, inactive_label="")) for name in result.detectors]
48
+ tiers.append(("timeline", result.timeline()))
49
+
50
+ lines = [
51
+ 'File type = "ooTextFile"',
52
+ 'Object class = "TextGrid"',
53
+ "",
54
+ f"xmin = {xmin}",
55
+ f"xmax = {xmax}",
56
+ "tiers? <exists>",
57
+ f"size = {len(tiers)}",
58
+ "item []:",
59
+ ]
60
+ for i, (name, segments) in enumerate(tiers, 1):
61
+ if not segments:
62
+ segments = [("", xmin, xmax)]
63
+ lines += [
64
+ f" item [{i}]:",
65
+ ' class = "IntervalTier"',
66
+ f" name = {_quote(name)}",
67
+ f" xmin = {xmin}",
68
+ f" xmax = {xmax}",
69
+ f" intervals: size = {len(segments)}",
70
+ ]
71
+ for j, (label, start, stop) in enumerate(segments, 1):
72
+ lines += [
73
+ f" intervals [{j}]:",
74
+ f" xmin = {start}",
75
+ f" xmax = {stop}",
76
+ f" text = {_quote(label)}",
77
+ ]
78
+ return "\n".join(lines) + "\n"
79
+
80
+
81
+ def export(result: Result, fmt: str, **metadata) -> str:
82
+ """Serialize ``result`` to one of :data:`FORMATS`."""
83
+ fmt = fmt.lower()
84
+ if fmt == "csv":
85
+ return to_csv(result)
86
+ if fmt == "tsv":
87
+ return to_csv(result, delimiter="\t")
88
+ if fmt == "json":
89
+ return to_json(result, **metadata)
90
+ if fmt == "textgrid":
91
+ return to_textgrid(result)
92
+ if fmt == "audacity":
93
+ return to_audacity(result)
94
+ raise ValueError(f"Unknown format {fmt!r}. Available: {', '.join(FORMATS)}")
inavamos/models.py ADDED
@@ -0,0 +1,210 @@
1
+ """Loading of the SSL encoders and downstream classifiers published on the HuggingFace Hub.
2
+
3
+ The models are loaded directly (without ``trust_remote_code``): the encoder is a
4
+ standard ``Data2VecAudioModel`` and the classifiers are small MLPs (:class:`FrameClassifier`)
5
+ stored as safetensors, or as ``torch.export`` programs in older revisions of the models.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import contextlib
11
+ import json
12
+ import logging
13
+ import os
14
+ from dataclasses import dataclass
15
+
16
+ import torch
17
+
18
+ logger = logging.getLogger(__name__)
19
+
20
+
21
+ @dataclass(frozen=True)
22
+ class DetectorSpec:
23
+ """Description of a detector published on the HuggingFace Hub."""
24
+
25
+ name: str
26
+ repo_id: str
27
+ # Viterbi self-transition probability used for smoothing.
28
+ transition: float
29
+ # The VAD classifier outputs the probability of *non*-speech.
30
+ invert: bool
31
+ # Input layout of the torch.export classifier (classifier/model.pt2), if used:
32
+ # "frames": (n_frames, n_layers, dim), "batch": (batch, n_layers, n_frames, dim).
33
+ layout: str
34
+
35
+
36
+ DETECTORS = {
37
+ "speech": DetectorSpec(
38
+ name="speech",
39
+ repo_id="ina-foss/ssl-vad-music2vec",
40
+ transition=0.99,
41
+ invert=True,
42
+ layout="frames",
43
+ ),
44
+ "music": DetectorSpec(
45
+ name="music",
46
+ repo_id="ina-foss/ssl-music-detection-music2vec",
47
+ transition=0.95,
48
+ invert=False,
49
+ layout="batch",
50
+ ),
51
+ }
52
+
53
+ DETECTOR_ALIASES = {"vad": "speech", "voice": "speech", "md": "music"}
54
+
55
+
56
+ def resolve_detector(name: str) -> str:
57
+ key = name.lower()
58
+ key = DETECTOR_ALIASES.get(key, key)
59
+ if key not in DETECTORS:
60
+ raise ValueError(f"Unknown detector {name!r}. Available: {', '.join(DETECTORS)}")
61
+ return key
62
+
63
+
64
+ class ModelAccessError(RuntimeError):
65
+ pass
66
+
67
+
68
+ def _access_message(repo_id: str) -> str:
69
+ return (
70
+ f"Cannot access the model '{repo_id}' on the HuggingFace Hub. Check the repository id,\n"
71
+ "and provide a token (HF_TOKEN, `hf auth login` or --token) if the repository is private."
72
+ )
73
+
74
+
75
+ def fetch_file(repo_id: str, filename: str, token=None, revision=None) -> str:
76
+ """Return a local path to ``filename`` from a Hub repository or a local directory."""
77
+ if os.path.isdir(repo_id):
78
+ path = os.path.join(repo_id, filename)
79
+ if not os.path.exists(path):
80
+ raise FileNotFoundError(path)
81
+ return path
82
+
83
+ from huggingface_hub import hf_hub_download
84
+ from huggingface_hub.errors import GatedRepoError, RepositoryNotFoundError
85
+
86
+ try:
87
+ return hf_hub_download(repo_id, filename, token=token, revision=revision)
88
+ except (GatedRepoError, RepositoryNotFoundError) as e:
89
+ raise ModelAccessError(_access_message(repo_id)) from e
90
+
91
+
92
+ def load_encoder(repo_id: str, token=None, revision=None, device="cpu"):
93
+ """Load the SSL encoder stored in the ``pretrained/`` subfolder of a repository."""
94
+ from transformers import Data2VecAudioConfig, Data2VecAudioModel
95
+
96
+ with open(fetch_file(repo_id, "pretrained/config.json", token, revision)) as f:
97
+ config_dict = json.load(f)
98
+ # The published configuration stores a path in `vocab_size`, which recent
99
+ # versions of transformers reject. It is unused by the audio encoder.
100
+ if not isinstance(config_dict.get("vocab_size"), int):
101
+ config_dict.pop("vocab_size", None)
102
+ config = Data2VecAudioConfig(**config_dict)
103
+
104
+ weights = fetch_file(repo_id, "pretrained/model.safetensors", token, revision)
105
+ with _quiet_transformers():
106
+ model = Data2VecAudioModel.from_pretrained(os.path.dirname(weights), config=config)
107
+ return model.to(device).eval()
108
+
109
+
110
+ def encoder_fingerprint(repo_id: str, token=None, revision=None) -> str:
111
+ """Identify an encoder by its weights so that identical encoders are only run once.
112
+
113
+ Files in the HuggingFace cache are symlinks to blobs named after their sha256.
114
+ """
115
+ weights = fetch_file(repo_id, "pretrained/model.safetensors", token, revision)
116
+ if os.path.islink(weights):
117
+ return os.path.basename(os.path.realpath(weights))
118
+ return os.path.abspath(weights)
119
+
120
+
121
+ @contextlib.contextmanager
122
+ def _quiet_transformers():
123
+ from transformers.utils import logging as hf_logging
124
+
125
+ enabled = hf_logging.is_progress_bar_enabled()
126
+ verbosity = hf_logging.get_verbosity()
127
+ hf_logging.disable_progress_bar()
128
+ hf_logging.set_verbosity_error()
129
+ try:
130
+ yield
131
+ finally:
132
+ hf_logging.set_verbosity(verbosity)
133
+ if enabled:
134
+ hf_logging.enable_progress_bar()
135
+
136
+
137
+ @contextlib.contextmanager
138
+ def _quiet_torch_export():
139
+ # The classifiers were serialized with an older torch.export format, which
140
+ # makes recent torch versions log a deprecation traceback on load.
141
+ torch_logger = logging.getLogger("torch.export")
142
+ level = torch_logger.level
143
+ torch_logger.setLevel(logging.ERROR)
144
+ try:
145
+ yield
146
+ finally:
147
+ torch_logger.setLevel(level)
148
+
149
+
150
+ class FrameClassifier(torch.nn.Module):
151
+ """MLP applied to each frame, on the concatenation of the encoder layers.
152
+
153
+ Maps features of shape (batch, n_layers, n_frames, dim) to logits of shape (batch, n_frames).
154
+ """
155
+
156
+ def __init__(self, input_size: int, hidden_sizes: list, batch_norm_eps: float = 1e-5):
157
+ super().__init__()
158
+ layers = []
159
+ size = input_size
160
+ for hidden_size in hidden_sizes:
161
+ layers += [
162
+ torch.nn.Linear(size, hidden_size),
163
+ torch.nn.BatchNorm1d(hidden_size, eps=batch_norm_eps),
164
+ torch.nn.ReLU(),
165
+ ]
166
+ size = hidden_size
167
+ self.hidden = torch.nn.Sequential(*layers)
168
+ self.out = torch.nn.Linear(size, 1)
169
+
170
+ def forward(self, features: torch.Tensor) -> torch.Tensor:
171
+ batch, _, n_frames, _ = features.shape
172
+ x = features.transpose(1, 2).reshape(batch * n_frames, -1)
173
+ return self.out(self.hidden(x)).reshape(batch, n_frames)
174
+
175
+
176
+ class ExportedClassifier(torch.nn.Module):
177
+ """Adapter giving a ``torch.export`` classifier the interface of :class:`FrameClassifier`."""
178
+
179
+ def __init__(self, module, layout: str):
180
+ super().__init__()
181
+ self.module = module
182
+ self.layout = layout
183
+
184
+ def forward(self, features: torch.Tensor) -> torch.Tensor:
185
+ if self.layout == "frames":
186
+ return torch.stack([self.module(f.transpose(0, 1)) for f in features])
187
+ return self.module(features)[0]
188
+
189
+
190
+ def load_classifier(spec: DetectorSpec, token=None, revision=None, device="cpu") -> torch.nn.Module:
191
+ """Load the classifier of a detector: ``classifier/model.safetensors`` if available,
192
+ otherwise the ``torch.export`` program ``classifier/model.pt2``."""
193
+ from huggingface_hub.errors import EntryNotFoundError
194
+
195
+ try:
196
+ config_path = fetch_file(spec.repo_id, "classifier/config.json", token, revision)
197
+ weights_path = fetch_file(spec.repo_id, "classifier/model.safetensors", token, revision)
198
+ except (EntryNotFoundError, FileNotFoundError):
199
+ path = fetch_file(spec.repo_id, "classifier/model.pt2", token, revision)
200
+ with _quiet_torch_export():
201
+ program = torch.export.load(path)
202
+ return ExportedClassifier(program.module(), spec.layout).to(device)
203
+
204
+ from safetensors.torch import load_file
205
+
206
+ with open(config_path) as f:
207
+ config = json.load(f)
208
+ classifier = FrameClassifier(config["input_size"], config["hidden_sizes"], config.get("batch_norm_eps", 1e-5))
209
+ classifier.load_state_dict(load_file(weights_path))
210
+ return classifier.to(device).eval()