inaVAMOS 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- inavamos/__init__.py +9 -0
- inavamos/__main__.py +5 -0
- inavamos/audio.py +57 -0
- inavamos/cli.py +132 -0
- inavamos/export.py +94 -0
- inavamos/models.py +210 -0
- inavamos/segmenter.py +320 -0
- inavamos-0.1.0.dist-info/METADATA +216 -0
- inavamos-0.1.0.dist-info/RECORD +13 -0
- inavamos-0.1.0.dist-info/WHEEL +4 -0
- inavamos-0.1.0.dist-info/entry_points.txt +2 -0
- inavamos-0.1.0.dist-info/licenses/LICENSE.en.md +155 -0
- inavamos-0.1.0.dist-info/licenses/LICENSE.md +175 -0
inavamos/__init__.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
"""inaVAMOS: Voice And Music Open Segmenter. Speech (voice activity) and music detection based on SSL models from INA."""
|
|
2
|
+
|
|
3
|
+
from .audio import load_audio
|
|
4
|
+
from .models import DETECTORS, ModelAccessError
|
|
5
|
+
from .segmenter import Result, Segment, Segmenter
|
|
6
|
+
|
|
7
|
+
__version__ = "0.1.0"
|
|
8
|
+
|
|
9
|
+
__all__ = ["Segmenter", "Segment", "Result", "load_audio", "DETECTORS", "ModelAccessError", "__version__"]
|
inavamos/__main__.py
ADDED
inavamos/audio.py
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"""Audio decoding to 16 kHz mono float32."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
import shutil
|
|
7
|
+
import subprocess
|
|
8
|
+
|
|
9
|
+
import numpy as np
|
|
10
|
+
|
|
11
|
+
SAMPLING_RATE = 16000
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def load_audio(path, sr: int = SAMPLING_RATE, start: float | None = None, stop: float | None = None) -> np.ndarray:
|
|
15
|
+
"""Decode any audio or video file to a mono float32 signal at ``sr`` Hz.
|
|
16
|
+
|
|
17
|
+
ffmpeg is used when available (it handles virtually every media format);
|
|
18
|
+
otherwise the file is decoded with librosa/soundfile.
|
|
19
|
+
|
|
20
|
+
Args:
|
|
21
|
+
path: media file path (or URL, when ffmpeg is used).
|
|
22
|
+
sr: target sampling rate.
|
|
23
|
+
start: optional start time, in seconds.
|
|
24
|
+
stop: optional stop time, in seconds.
|
|
25
|
+
"""
|
|
26
|
+
if start is not None and start < 0:
|
|
27
|
+
raise ValueError("start must be positive")
|
|
28
|
+
if start is not None and stop is not None and stop <= start:
|
|
29
|
+
raise ValueError("stop must be greater than start")
|
|
30
|
+
if "://" not in str(path) and not os.path.exists(path):
|
|
31
|
+
raise FileNotFoundError(f"No such file: {path}")
|
|
32
|
+
|
|
33
|
+
if shutil.which("ffmpeg"):
|
|
34
|
+
return _load_ffmpeg(path, sr, start, stop)
|
|
35
|
+
return _load_librosa(path, sr, start, stop)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _load_ffmpeg(path, sr, start, stop):
|
|
39
|
+
cmd = ["ffmpeg", "-nostdin", "-hide_banner", "-loglevel", "error"]
|
|
40
|
+
if start:
|
|
41
|
+
cmd += ["-ss", str(start)]
|
|
42
|
+
cmd += ["-i", str(path)]
|
|
43
|
+
if stop is not None:
|
|
44
|
+
cmd += ["-t", str(stop - (start or 0))]
|
|
45
|
+
cmd += ["-map", "0:a:0", "-vn", "-ac", "1", "-ar", str(sr), "-f", "f32le", "-acodec", "pcm_f32le", "-"]
|
|
46
|
+
proc = subprocess.run(cmd, capture_output=True)
|
|
47
|
+
if proc.returncode != 0:
|
|
48
|
+
raise RuntimeError(f"ffmpeg failed to decode {path}:\n{proc.stderr.decode(errors='replace').strip()}")
|
|
49
|
+
return np.frombuffer(proc.stdout, dtype=np.float32).copy()
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _load_librosa(path, sr, start, stop):
|
|
53
|
+
import librosa
|
|
54
|
+
|
|
55
|
+
duration = None if stop is None else stop - (start or 0)
|
|
56
|
+
audio, _ = librosa.load(path, sr=sr, mono=True, offset=start or 0.0, duration=duration)
|
|
57
|
+
return audio.astype(np.float32)
|
inavamos/cli.py
ADDED
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
"""Command line interface: ``ina-vamos -i media.mp3 -o output_dir``."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import logging
|
|
7
|
+
import sys
|
|
8
|
+
import time
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
from . import __version__
|
|
12
|
+
from .export import FORMATS, export
|
|
13
|
+
from .models import DETECTORS, ModelAccessError
|
|
14
|
+
from .segmenter import Segmenter
|
|
15
|
+
|
|
16
|
+
logger = logging.getLogger("inavamos")
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
20
|
+
parser = argparse.ArgumentParser(
|
|
21
|
+
prog="ina-vamos",
|
|
22
|
+
description=(
|
|
23
|
+
"inaVAMOS (Voice And Music Open Segmenter): speech (voice activity) and music "
|
|
24
|
+
"detection in audio and video files, using SSL-based models from INA (https://huggingface.co/ina-foss)."
|
|
25
|
+
),
|
|
26
|
+
)
|
|
27
|
+
parser.add_argument("-i", "--input", nargs="+", required=True, help="media files to process (any format read by ffmpeg)")
|
|
28
|
+
parser.add_argument(
|
|
29
|
+
"-o", "--output-dir", type=Path, help="directory where results are written (default: print to standard output)"
|
|
30
|
+
)
|
|
31
|
+
parser.add_argument("-f", "--format", choices=list(FORMATS), default="csv", help="output format (default: csv)")
|
|
32
|
+
parser.add_argument(
|
|
33
|
+
"-d",
|
|
34
|
+
"--detectors",
|
|
35
|
+
nargs="+",
|
|
36
|
+
default=list(DETECTORS),
|
|
37
|
+
metavar="{speech,music}",
|
|
38
|
+
help="detectors to run (default: speech music). 'vad' is an alias of 'speech'",
|
|
39
|
+
)
|
|
40
|
+
parser.add_argument("--device", help="torch device, e.g. cpu, cuda, cuda:1 (default: cuda if available)")
|
|
41
|
+
parser.add_argument("--batch-size", type=int, default=8, help="number of windows processed together (default: 8)")
|
|
42
|
+
parser.add_argument(
|
|
43
|
+
"--window-duration",
|
|
44
|
+
type=float,
|
|
45
|
+
default=30.0,
|
|
46
|
+
help="duration in seconds of the analysis windows (default: 30). 0 processes files at once",
|
|
47
|
+
)
|
|
48
|
+
parser.add_argument(
|
|
49
|
+
"--context-duration",
|
|
50
|
+
type=float,
|
|
51
|
+
default=2.5,
|
|
52
|
+
help="overlap in seconds on each side of the windows (default: 2.5)",
|
|
53
|
+
)
|
|
54
|
+
parser.add_argument("--start", type=float, help="start time in seconds of the portion to process")
|
|
55
|
+
parser.add_argument("--stop", type=float, help="stop time in seconds of the portion to process")
|
|
56
|
+
parser.add_argument("--token", help="HuggingFace token, only needed for private model repositories")
|
|
57
|
+
parser.add_argument("--revision", help="revision of the models on the HuggingFace Hub")
|
|
58
|
+
parser.add_argument("--overwrite", action="store_true", help="overwrite existing output files instead of skipping them")
|
|
59
|
+
parser.add_argument("-v", "--verbose", action="store_true", help="print progress information")
|
|
60
|
+
parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
|
|
61
|
+
return parser
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def main(argv=None) -> int:
|
|
65
|
+
args = build_parser().parse_args(argv)
|
|
66
|
+
logging.basicConfig(format="%(levelname)s: %(message)s", level=logging.INFO if args.verbose else logging.WARNING)
|
|
67
|
+
logging.getLogger("httpx").setLevel(logging.WARNING)
|
|
68
|
+
|
|
69
|
+
inputs = [Path(p) for p in args.input]
|
|
70
|
+
outputs = {}
|
|
71
|
+
if args.output_dir:
|
|
72
|
+
args.output_dir.mkdir(parents=True, exist_ok=True)
|
|
73
|
+
for path in inputs:
|
|
74
|
+
output = args.output_dir / (path.stem + FORMATS[args.format])
|
|
75
|
+
if output in outputs.values():
|
|
76
|
+
logger.error("Several inputs would be written to %s", output)
|
|
77
|
+
return 2
|
|
78
|
+
outputs[path] = output
|
|
79
|
+
if not args.overwrite:
|
|
80
|
+
todo = [p for p in inputs if not outputs[p].exists()]
|
|
81
|
+
if len(todo) < len(inputs):
|
|
82
|
+
logger.warning("Skipping %d files already processed (use --overwrite)", len(inputs) - len(todo))
|
|
83
|
+
inputs = todo
|
|
84
|
+
if not inputs:
|
|
85
|
+
return 0
|
|
86
|
+
|
|
87
|
+
try:
|
|
88
|
+
segmenter = Segmenter(
|
|
89
|
+
detectors=args.detectors,
|
|
90
|
+
device=args.device,
|
|
91
|
+
window_duration=args.window_duration or None,
|
|
92
|
+
context_duration=args.context_duration,
|
|
93
|
+
batch_size=args.batch_size,
|
|
94
|
+
token=args.token,
|
|
95
|
+
revision=args.revision,
|
|
96
|
+
)
|
|
97
|
+
except (ModelAccessError, ValueError) as e:
|
|
98
|
+
logger.error("%s", e)
|
|
99
|
+
return 2
|
|
100
|
+
|
|
101
|
+
repo_ids = {name: spec.repo_id for name, spec in segmenter.specs.items()}
|
|
102
|
+
failures = 0
|
|
103
|
+
for i, path in enumerate(inputs, 1):
|
|
104
|
+
t0 = time.perf_counter()
|
|
105
|
+
try:
|
|
106
|
+
result = segmenter.process(path, start=args.start, stop=args.stop)
|
|
107
|
+
except Exception as e: # keep processing the other files
|
|
108
|
+
logger.error("%s: %s", path, e)
|
|
109
|
+
failures += 1
|
|
110
|
+
continue
|
|
111
|
+
elapsed = time.perf_counter() - t0
|
|
112
|
+
logger.info(
|
|
113
|
+
"[%d/%d] %s: %.1fs of audio in %.1fs (%.0fx real time)",
|
|
114
|
+
i,
|
|
115
|
+
len(inputs),
|
|
116
|
+
path,
|
|
117
|
+
result.duration,
|
|
118
|
+
elapsed,
|
|
119
|
+
result.duration / max(elapsed, 1e-6),
|
|
120
|
+
)
|
|
121
|
+
content = export(result, args.format, file=str(path), models=repo_ids)
|
|
122
|
+
if args.output_dir:
|
|
123
|
+
outputs[path].write_text(content)
|
|
124
|
+
else:
|
|
125
|
+
if len(inputs) > 1:
|
|
126
|
+
print(f"# {path}")
|
|
127
|
+
sys.stdout.write(content)
|
|
128
|
+
return 1 if failures else 0
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
if __name__ == "__main__":
|
|
132
|
+
sys.exit(main())
|
inavamos/export.py
ADDED
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
"""Export of segmentation results to common file formats."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import csv
|
|
6
|
+
import io
|
|
7
|
+
import json
|
|
8
|
+
|
|
9
|
+
from .segmenter import Result
|
|
10
|
+
|
|
11
|
+
FORMATS = {
|
|
12
|
+
"csv": ".csv",
|
|
13
|
+
"tsv": ".tsv",
|
|
14
|
+
"json": ".json",
|
|
15
|
+
"textgrid": ".TextGrid",
|
|
16
|
+
"audacity": ".txt",
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def to_csv(result: Result, delimiter: str = ",") -> str:
|
|
21
|
+
"""Combined timeline as a table with ``label,start,stop`` columns."""
|
|
22
|
+
out = io.StringIO()
|
|
23
|
+
writer = csv.writer(out, delimiter=delimiter, lineterminator="\n")
|
|
24
|
+
writer.writerow(["label", "start", "stop"])
|
|
25
|
+
for label, start, stop in result.timeline():
|
|
26
|
+
writer.writerow([label, f"{start:.3f}", f"{stop:.3f}"])
|
|
27
|
+
return out.getvalue()
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def to_json(result: Result, **metadata) -> str:
|
|
31
|
+
"""Per-detector segments and combined timeline as JSON."""
|
|
32
|
+
return json.dumps({**metadata, **result.to_dict()}, indent=2)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def to_audacity(result: Result) -> str:
|
|
36
|
+
"""Combined timeline as an Audacity label track (File > Import > Labels)."""
|
|
37
|
+
return "".join(f"{start:.6f}\t{stop:.6f}\t{label}\n" for label, start, stop in result.timeline())
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _quote(text: str) -> str:
|
|
41
|
+
return '"' + text.replace('"', '""') + '"'
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def to_textgrid(result: Result) -> str:
|
|
45
|
+
"""Praat TextGrid with one tier per detector and a tier for the combined timeline."""
|
|
46
|
+
xmin, xmax = result.offset, result.offset + result.duration
|
|
47
|
+
tiers = [(name, result.segments(name, inactive_label="")) for name in result.detectors]
|
|
48
|
+
tiers.append(("timeline", result.timeline()))
|
|
49
|
+
|
|
50
|
+
lines = [
|
|
51
|
+
'File type = "ooTextFile"',
|
|
52
|
+
'Object class = "TextGrid"',
|
|
53
|
+
"",
|
|
54
|
+
f"xmin = {xmin}",
|
|
55
|
+
f"xmax = {xmax}",
|
|
56
|
+
"tiers? <exists>",
|
|
57
|
+
f"size = {len(tiers)}",
|
|
58
|
+
"item []:",
|
|
59
|
+
]
|
|
60
|
+
for i, (name, segments) in enumerate(tiers, 1):
|
|
61
|
+
if not segments:
|
|
62
|
+
segments = [("", xmin, xmax)]
|
|
63
|
+
lines += [
|
|
64
|
+
f" item [{i}]:",
|
|
65
|
+
' class = "IntervalTier"',
|
|
66
|
+
f" name = {_quote(name)}",
|
|
67
|
+
f" xmin = {xmin}",
|
|
68
|
+
f" xmax = {xmax}",
|
|
69
|
+
f" intervals: size = {len(segments)}",
|
|
70
|
+
]
|
|
71
|
+
for j, (label, start, stop) in enumerate(segments, 1):
|
|
72
|
+
lines += [
|
|
73
|
+
f" intervals [{j}]:",
|
|
74
|
+
f" xmin = {start}",
|
|
75
|
+
f" xmax = {stop}",
|
|
76
|
+
f" text = {_quote(label)}",
|
|
77
|
+
]
|
|
78
|
+
return "\n".join(lines) + "\n"
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def export(result: Result, fmt: str, **metadata) -> str:
|
|
82
|
+
"""Serialize ``result`` to one of :data:`FORMATS`."""
|
|
83
|
+
fmt = fmt.lower()
|
|
84
|
+
if fmt == "csv":
|
|
85
|
+
return to_csv(result)
|
|
86
|
+
if fmt == "tsv":
|
|
87
|
+
return to_csv(result, delimiter="\t")
|
|
88
|
+
if fmt == "json":
|
|
89
|
+
return to_json(result, **metadata)
|
|
90
|
+
if fmt == "textgrid":
|
|
91
|
+
return to_textgrid(result)
|
|
92
|
+
if fmt == "audacity":
|
|
93
|
+
return to_audacity(result)
|
|
94
|
+
raise ValueError(f"Unknown format {fmt!r}. Available: {', '.join(FORMATS)}")
|
inavamos/models.py
ADDED
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
"""Loading of the SSL encoders and downstream classifiers published on the HuggingFace Hub.
|
|
2
|
+
|
|
3
|
+
The models are loaded directly (without ``trust_remote_code``): the encoder is a
|
|
4
|
+
standard ``Data2VecAudioModel`` and the classifiers are small MLPs (:class:`FrameClassifier`)
|
|
5
|
+
stored as safetensors, or as ``torch.export`` programs in older revisions of the models.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import contextlib
|
|
11
|
+
import json
|
|
12
|
+
import logging
|
|
13
|
+
import os
|
|
14
|
+
from dataclasses import dataclass
|
|
15
|
+
|
|
16
|
+
import torch
|
|
17
|
+
|
|
18
|
+
logger = logging.getLogger(__name__)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass(frozen=True)
|
|
22
|
+
class DetectorSpec:
|
|
23
|
+
"""Description of a detector published on the HuggingFace Hub."""
|
|
24
|
+
|
|
25
|
+
name: str
|
|
26
|
+
repo_id: str
|
|
27
|
+
# Viterbi self-transition probability used for smoothing.
|
|
28
|
+
transition: float
|
|
29
|
+
# The VAD classifier outputs the probability of *non*-speech.
|
|
30
|
+
invert: bool
|
|
31
|
+
# Input layout of the torch.export classifier (classifier/model.pt2), if used:
|
|
32
|
+
# "frames": (n_frames, n_layers, dim), "batch": (batch, n_layers, n_frames, dim).
|
|
33
|
+
layout: str
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
DETECTORS = {
|
|
37
|
+
"speech": DetectorSpec(
|
|
38
|
+
name="speech",
|
|
39
|
+
repo_id="ina-foss/ssl-vad-music2vec",
|
|
40
|
+
transition=0.99,
|
|
41
|
+
invert=True,
|
|
42
|
+
layout="frames",
|
|
43
|
+
),
|
|
44
|
+
"music": DetectorSpec(
|
|
45
|
+
name="music",
|
|
46
|
+
repo_id="ina-foss/ssl-music-detection-music2vec",
|
|
47
|
+
transition=0.95,
|
|
48
|
+
invert=False,
|
|
49
|
+
layout="batch",
|
|
50
|
+
),
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
DETECTOR_ALIASES = {"vad": "speech", "voice": "speech", "md": "music"}
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def resolve_detector(name: str) -> str:
|
|
57
|
+
key = name.lower()
|
|
58
|
+
key = DETECTOR_ALIASES.get(key, key)
|
|
59
|
+
if key not in DETECTORS:
|
|
60
|
+
raise ValueError(f"Unknown detector {name!r}. Available: {', '.join(DETECTORS)}")
|
|
61
|
+
return key
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class ModelAccessError(RuntimeError):
|
|
65
|
+
pass
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _access_message(repo_id: str) -> str:
|
|
69
|
+
return (
|
|
70
|
+
f"Cannot access the model '{repo_id}' on the HuggingFace Hub. Check the repository id,\n"
|
|
71
|
+
"and provide a token (HF_TOKEN, `hf auth login` or --token) if the repository is private."
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def fetch_file(repo_id: str, filename: str, token=None, revision=None) -> str:
|
|
76
|
+
"""Return a local path to ``filename`` from a Hub repository or a local directory."""
|
|
77
|
+
if os.path.isdir(repo_id):
|
|
78
|
+
path = os.path.join(repo_id, filename)
|
|
79
|
+
if not os.path.exists(path):
|
|
80
|
+
raise FileNotFoundError(path)
|
|
81
|
+
return path
|
|
82
|
+
|
|
83
|
+
from huggingface_hub import hf_hub_download
|
|
84
|
+
from huggingface_hub.errors import GatedRepoError, RepositoryNotFoundError
|
|
85
|
+
|
|
86
|
+
try:
|
|
87
|
+
return hf_hub_download(repo_id, filename, token=token, revision=revision)
|
|
88
|
+
except (GatedRepoError, RepositoryNotFoundError) as e:
|
|
89
|
+
raise ModelAccessError(_access_message(repo_id)) from e
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def load_encoder(repo_id: str, token=None, revision=None, device="cpu"):
|
|
93
|
+
"""Load the SSL encoder stored in the ``pretrained/`` subfolder of a repository."""
|
|
94
|
+
from transformers import Data2VecAudioConfig, Data2VecAudioModel
|
|
95
|
+
|
|
96
|
+
with open(fetch_file(repo_id, "pretrained/config.json", token, revision)) as f:
|
|
97
|
+
config_dict = json.load(f)
|
|
98
|
+
# The published configuration stores a path in `vocab_size`, which recent
|
|
99
|
+
# versions of transformers reject. It is unused by the audio encoder.
|
|
100
|
+
if not isinstance(config_dict.get("vocab_size"), int):
|
|
101
|
+
config_dict.pop("vocab_size", None)
|
|
102
|
+
config = Data2VecAudioConfig(**config_dict)
|
|
103
|
+
|
|
104
|
+
weights = fetch_file(repo_id, "pretrained/model.safetensors", token, revision)
|
|
105
|
+
with _quiet_transformers():
|
|
106
|
+
model = Data2VecAudioModel.from_pretrained(os.path.dirname(weights), config=config)
|
|
107
|
+
return model.to(device).eval()
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def encoder_fingerprint(repo_id: str, token=None, revision=None) -> str:
|
|
111
|
+
"""Identify an encoder by its weights so that identical encoders are only run once.
|
|
112
|
+
|
|
113
|
+
Files in the HuggingFace cache are symlinks to blobs named after their sha256.
|
|
114
|
+
"""
|
|
115
|
+
weights = fetch_file(repo_id, "pretrained/model.safetensors", token, revision)
|
|
116
|
+
if os.path.islink(weights):
|
|
117
|
+
return os.path.basename(os.path.realpath(weights))
|
|
118
|
+
return os.path.abspath(weights)
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
@contextlib.contextmanager
|
|
122
|
+
def _quiet_transformers():
|
|
123
|
+
from transformers.utils import logging as hf_logging
|
|
124
|
+
|
|
125
|
+
enabled = hf_logging.is_progress_bar_enabled()
|
|
126
|
+
verbosity = hf_logging.get_verbosity()
|
|
127
|
+
hf_logging.disable_progress_bar()
|
|
128
|
+
hf_logging.set_verbosity_error()
|
|
129
|
+
try:
|
|
130
|
+
yield
|
|
131
|
+
finally:
|
|
132
|
+
hf_logging.set_verbosity(verbosity)
|
|
133
|
+
if enabled:
|
|
134
|
+
hf_logging.enable_progress_bar()
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
@contextlib.contextmanager
|
|
138
|
+
def _quiet_torch_export():
|
|
139
|
+
# The classifiers were serialized with an older torch.export format, which
|
|
140
|
+
# makes recent torch versions log a deprecation traceback on load.
|
|
141
|
+
torch_logger = logging.getLogger("torch.export")
|
|
142
|
+
level = torch_logger.level
|
|
143
|
+
torch_logger.setLevel(logging.ERROR)
|
|
144
|
+
try:
|
|
145
|
+
yield
|
|
146
|
+
finally:
|
|
147
|
+
torch_logger.setLevel(level)
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
class FrameClassifier(torch.nn.Module):
|
|
151
|
+
"""MLP applied to each frame, on the concatenation of the encoder layers.
|
|
152
|
+
|
|
153
|
+
Maps features of shape (batch, n_layers, n_frames, dim) to logits of shape (batch, n_frames).
|
|
154
|
+
"""
|
|
155
|
+
|
|
156
|
+
def __init__(self, input_size: int, hidden_sizes: list, batch_norm_eps: float = 1e-5):
|
|
157
|
+
super().__init__()
|
|
158
|
+
layers = []
|
|
159
|
+
size = input_size
|
|
160
|
+
for hidden_size in hidden_sizes:
|
|
161
|
+
layers += [
|
|
162
|
+
torch.nn.Linear(size, hidden_size),
|
|
163
|
+
torch.nn.BatchNorm1d(hidden_size, eps=batch_norm_eps),
|
|
164
|
+
torch.nn.ReLU(),
|
|
165
|
+
]
|
|
166
|
+
size = hidden_size
|
|
167
|
+
self.hidden = torch.nn.Sequential(*layers)
|
|
168
|
+
self.out = torch.nn.Linear(size, 1)
|
|
169
|
+
|
|
170
|
+
def forward(self, features: torch.Tensor) -> torch.Tensor:
|
|
171
|
+
batch, _, n_frames, _ = features.shape
|
|
172
|
+
x = features.transpose(1, 2).reshape(batch * n_frames, -1)
|
|
173
|
+
return self.out(self.hidden(x)).reshape(batch, n_frames)
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
class ExportedClassifier(torch.nn.Module):
|
|
177
|
+
"""Adapter giving a ``torch.export`` classifier the interface of :class:`FrameClassifier`."""
|
|
178
|
+
|
|
179
|
+
def __init__(self, module, layout: str):
|
|
180
|
+
super().__init__()
|
|
181
|
+
self.module = module
|
|
182
|
+
self.layout = layout
|
|
183
|
+
|
|
184
|
+
def forward(self, features: torch.Tensor) -> torch.Tensor:
|
|
185
|
+
if self.layout == "frames":
|
|
186
|
+
return torch.stack([self.module(f.transpose(0, 1)) for f in features])
|
|
187
|
+
return self.module(features)[0]
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def load_classifier(spec: DetectorSpec, token=None, revision=None, device="cpu") -> torch.nn.Module:
|
|
191
|
+
"""Load the classifier of a detector: ``classifier/model.safetensors`` if available,
|
|
192
|
+
otherwise the ``torch.export`` program ``classifier/model.pt2``."""
|
|
193
|
+
from huggingface_hub.errors import EntryNotFoundError
|
|
194
|
+
|
|
195
|
+
try:
|
|
196
|
+
config_path = fetch_file(spec.repo_id, "classifier/config.json", token, revision)
|
|
197
|
+
weights_path = fetch_file(spec.repo_id, "classifier/model.safetensors", token, revision)
|
|
198
|
+
except (EntryNotFoundError, FileNotFoundError):
|
|
199
|
+
path = fetch_file(spec.repo_id, "classifier/model.pt2", token, revision)
|
|
200
|
+
with _quiet_torch_export():
|
|
201
|
+
program = torch.export.load(path)
|
|
202
|
+
return ExportedClassifier(program.module(), spec.layout).to(device)
|
|
203
|
+
|
|
204
|
+
from safetensors.torch import load_file
|
|
205
|
+
|
|
206
|
+
with open(config_path) as f:
|
|
207
|
+
config = json.load(f)
|
|
208
|
+
classifier = FrameClassifier(config["input_size"], config["hidden_sizes"], config.get("batch_norm_eps", 1e-5))
|
|
209
|
+
classifier.load_state_dict(load_file(weights_path))
|
|
210
|
+
return classifier.to(device).eval()
|