DeePAQ 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
deepaq/__init__.py ADDED
@@ -0,0 +1,43 @@
1
+ """DeePAQ: MERT embeddings for audio quality assessment."""
2
+
3
+ __version__ = "0.1.0"
4
+
5
+
6
+ def compute_embedding(
7
+ wav_path,
8
+ checkpoint,
9
+ model_type="MERT",
10
+ mapping_type="polynomial",
11
+ cuda_device=0,
12
+ sampling_rate=24000,
13
+ mapping_model_path=None,
14
+ ):
15
+ """Return the 256-d quality embedding for one audio file.
16
+
17
+ ``mapping_type`` is accepted so the documented call works. The embedding
18
+ itself does not depend on the distance-to-score mapping. Mapped scores for
19
+ a test set are produced by the ``deepaq`` command. ``mapping_model_path``
20
+ is required only when ``mapping_type`` is ``"nn"``.
21
+ """
22
+ import pandas as pd
23
+
24
+ from deepaq.train.training import Evaluation
25
+
26
+ if mapping_type == "nn" and not mapping_model_path:
27
+ raise ValueError("mapping_model_path is required when mapping_type is 'nn'.")
28
+
29
+ config = {
30
+ "model": model_type,
31
+ "model_path": checkpoint,
32
+ "ssl_out_dim": 768,
33
+ "emb_dim": 256,
34
+ "sampling_rate": sampling_rate,
35
+ "cuda_device": cuda_device,
36
+ "mapping": mapping_type,
37
+ }
38
+ if mapping_model_path:
39
+ config["mapping_model_path"] = mapping_model_path
40
+ evaluator = Evaluation(config)
41
+ names = pd.Series([wav_path], name="filepath")
42
+ frame = evaluator.get_embeddings_csv(names, root=None)
43
+ return frame.iloc[0, 1:].to_numpy()
deepaq/cli.py ADDED
@@ -0,0 +1,25 @@
1
+ import click
2
+ import yaml
3
+
4
+ from deepaq.train.training import Evaluation, Training
5
+
6
+
7
+ @click.command()
8
+ @click.option("--config_file", type=str, required=True)
9
+ def main(config_file):
10
+ with open(config_file) as file:
11
+ config = yaml.load(file, Loader=yaml.FullLoader)
12
+
13
+ experiment = config["experiment_name"]
14
+ if experiment == "Training":
15
+ Training(config).training_loop()
16
+ elif experiment == "Full_ref_embedding_analysis":
17
+ Evaluation(config).eval_embedding_full_ref()
18
+ elif experiment == "Non_matching_ref_embedding_analysis":
19
+ Evaluation(config).eval_embedding_non_matching_ref()
20
+ else:
21
+ raise click.ClickException(f"Unknown experiment_name '{experiment}'.")
22
+
23
+
24
+ if __name__ == "__main__":
25
+ main()
File without changes
@@ -0,0 +1,89 @@
1
+ from torch.utils.data import Dataset, Sampler
2
+ import pandas as pd
3
+ import os
4
+ import torchaudio
5
+ import torch
6
+ import numpy as np
7
+
8
+
9
+ def load_processing(filepath, target_sr=24000, trim=False, wav_norm=False):
10
+ if isinstance(filepath, np.ndarray):
11
+ filepath = filepath[0]
12
+ wave, sr = torchaudio.load(filepath)
13
+
14
+ if wave.shape[0] > 1:
15
+ wave = ((wave[0, :] + wave[1, :]) / 2).unsqueeze(0)
16
+
17
+ if sr != target_sr:
18
+ wave = torchaudio.transforms.Resample(sr, target_sr)(wave)
19
+ sr = target_sr
20
+
21
+ if trim and wave.shape[1] > sr * 5:
22
+ wave = wave[:, :sr * 5]
23
+
24
+ if wav_norm:
25
+ energy = torch.mean(torch.abs(wave - torch.mean(wave)) ** 2, dtype=float)
26
+ wave = (wave - torch.mean(wave)) / torch.sqrt(energy)
27
+
28
+ return wave
29
+
30
+
31
+ class WSDataset(Dataset):
32
+ def __init__(self, config, data_mode='train_df'):
33
+ super().__init__()
34
+ self.config = config
35
+ self.root = self.config['root_train'] if data_mode == 'train_df' else self.config['root_valid']
36
+ self.dataset = pd.read_csv(self.config[data_mode]).drop_duplicates()
37
+ self.label_to_index = {
38
+ 'ref': 0,
39
+ 'he-aac': 1,
40
+ 'mp3': 2,
41
+ 'opus': 3,
42
+ 'speech': 4,
43
+ }
44
+ self.bitrate_to_index = {
45
+ '16000': 1.5,
46
+ '32000': 2.0,
47
+ '48000': 2.5,
48
+ '64000': 3.0,
49
+ '80000': 3.5,
50
+ '96000': 4.0,
51
+ '128000': 4.5,
52
+ 'ref': 5,
53
+ }
54
+
55
+ def __len__(self):
56
+ return len(self.dataset)
57
+
58
+ def __getitem__(self, index):
59
+ row = self.dataset.iloc[index]
60
+ wav = load_processing(
61
+ os.path.join(self.root, row['Item']),
62
+ target_sr=self.config['sampling_rate'],
63
+ trim=self.config['trim'],
64
+ wav_norm=self.config['wav_norm'],
65
+ )
66
+ label = self.label_to_index[row['Distortion']]
67
+ if label == 4:
68
+ bitrate_label = float(row['Bitrate'])
69
+ else:
70
+ bitrate_label = self.bitrate_to_index[str(row['Bitrate'])]
71
+ return wav, label, row['Distortion_Intensity'], bitrate_label
72
+
73
+ def collate_fn(self, batch):
74
+ waves, labels, sub_labels, bitrate_labels = zip(*batch)
75
+ waves = self.zero_pad_wav(waves)
76
+ labels = torch.tensor(labels, dtype=torch.long)
77
+ sub_labels = torch.tensor([float(sub_label) for sub_label in sub_labels], dtype=torch.float32)
78
+ bitrate_labels = torch.tensor(bitrate_labels, dtype=torch.float32)
79
+ return waves, labels, sub_labels, bitrate_labels
80
+
81
+ def zero_pad_wav(self, wavs):
82
+ max_len = self.config['sampling_rate'] * 5
83
+ padded = [
84
+ torch.nn.functional.pad(wav, (0, max_len - wav.shape[1]), 'constant', 0)
85
+ for wav in wavs
86
+ ]
87
+ return torch.stack(padded, dim=0)
88
+
89
+
File without changes
deepaq/losses/loss.py ADDED
@@ -0,0 +1,58 @@
1
+ import torch
2
+ import torch.nn as nn
3
+ import numpy as np
4
+
5
+
6
+ class RnC_Loss(nn.Module):
7
+ def __init__(self, config):
8
+ super().__init__()
9
+ self.config = config
10
+
11
+ def forward(self, embeddings, labels, sub_labels, bitrate_labels):
12
+ loss = torch.zeros((), device=embeddings.device)
13
+ if self.config['sorted']:
14
+ mask = self.get_mask(labels).to(embeddings.device)
15
+ for group in mask:
16
+ idx = torch.where(group)[0]
17
+ if idx.numel() > 2:
18
+ loss = loss + self.config['lambda'] * self.rnc_loss(embeddings[idx], bitrate_labels[idx])
19
+ return loss + self.rnc_loss(embeddings, sub_labels)
20
+
21
+ @staticmethod
22
+ def get_mask(labels):
23
+ """Boolean groups of reference samples plus each non-reference label."""
24
+ labels = labels.cpu().data.numpy()
25
+ groups = []
26
+ for label in set(labels):
27
+ indices = np.where((labels == label) & (labels != 0))[0]
28
+ if len(indices) >= 2:
29
+ groups.append(indices)
30
+
31
+ ref_indices = set(np.where(labels == 0)[0])
32
+ groups = [list(ref_indices.union(group)) for group in groups]
33
+
34
+ mask = torch.zeros((len(groups), len(labels)), dtype=torch.bool)
35
+ for i, group in enumerate(groups):
36
+ mask[i, group] = True
37
+ return mask
38
+
39
+ @staticmethod
40
+ def rnc_loss(embeddings, targets):
41
+ label_diffs = torch.abs(targets[:, None] - targets[None, :])
42
+ logits = -(embeddings[:, None, :] - embeddings[None, :, :]).norm(2, dim=-1)
43
+ logits = logits - logits.max(dim=1, keepdim=True).values.detach()
44
+ exp_logits = logits.exp()
45
+
46
+ n = logits.shape[0]
47
+ off_diag = ~torch.eye(n, dtype=torch.bool, device=logits.device)
48
+ logits = logits[off_diag].view(n, n - 1)
49
+ exp_logits = exp_logits[off_diag].view(n, n - 1)
50
+ label_diffs = label_diffs[off_diag].view(n, n - 1)
51
+
52
+ loss = logits.new_zeros(())
53
+ for k in range(n - 1):
54
+ pos_logits = logits[:, k]
55
+ neg_mask = (label_diffs >= label_diffs[:, k].view(-1, 1)).float()
56
+ pos_log_probs = pos_logits - torch.log((neg_mask * exp_logits).sum(dim=-1))
57
+ loss = loss - (pos_log_probs / (n * (n - 1))).sum()
58
+ return loss
File without changes
@@ -0,0 +1,36 @@
1
+ import torch
2
+ import torch.nn as nn
3
+
4
+
5
+ class MERT4AudioQuality(nn.Module):
6
+ def __init__(self, ssl_model, ssl_out_dim, emb_dim=768):
7
+ super().__init__()
8
+ self.ssl_model = ssl_model
9
+ self.embedding_layer = nn.Sequential(
10
+ nn.ReLU(),
11
+ nn.Linear(ssl_out_dim * 13, emb_dim),
12
+ )
13
+
14
+ def forward(self, wav):
15
+ wav = wav.squeeze(1)
16
+ hidden_states = torch.stack(self.ssl_model(wav, output_hidden_states=True).hidden_states)
17
+ # Utterance-level embedding: average over time, then concatenate layers.
18
+ hidden_states = hidden_states.mean(-2).permute(1, 0, 2)
19
+ hidden_states = hidden_states.reshape(hidden_states.size(0), -1)
20
+ return self.embedding_layer(hidden_states)
21
+
22
+
23
+ class DistanceToScoreNet(nn.Module):
24
+ def __init__(self, emb_dim=1):
25
+ super().__init__()
26
+ self.layers = nn.Sequential(
27
+ nn.Linear(emb_dim, 64),
28
+ nn.ReLU(),
29
+ nn.Linear(64, 16),
30
+ nn.ReLU(),
31
+ nn.Linear(16, 1),
32
+ nn.Sigmoid(),
33
+ )
34
+
35
+ def forward(self, x):
36
+ return 100 * self.layers(x)
File without changes
@@ -0,0 +1,384 @@
1
+ import glob
2
+ import os
3
+ import random
4
+ from datetime import datetime
5
+
6
+ import matplotlib.pyplot as plt
7
+ import numpy as np
8
+ import pandas as pd
9
+ import seaborn as sns
10
+ import torch
11
+ import torch.optim.lr_scheduler as lr_scheduler
12
+ import yaml
13
+ from peft.tuners.lora import Linear as LoRALinear
14
+ from scipy.interpolate import LSQUnivariateSpline
15
+ from scipy.optimize import curve_fit
16
+ from scipy.spatial.distance import cdist
17
+ from scipy.stats import linregress, pearsonr, spearmanr
18
+ from torch.utils.data import DataLoader
19
+ from tqdm import tqdm
20
+ from transformers import AutoModel
21
+
22
+ from deepaq.data.dataloader import WSDataset, load_processing
23
+ from deepaq.losses.loss import RnC_Loss
24
+ from deepaq.networks.network import DistanceToScoreNet, MERT4AudioQuality
25
+
26
+ import warnings
27
+ warnings.filterwarnings("ignore", category=UserWarning)
28
+ sns.set_style('darkgrid')
29
+
30
+ SEED = 0
31
+ torch.manual_seed(SEED)
32
+ np.random.seed(SEED)
33
+ random.seed(SEED)
34
+ torch.backends.cudnn.deterministic = True
35
+ torch.backends.cudnn.benchmark = False
36
+
37
+ MERT_MODEL_NAME = "m-a-p/MERT-v1-95M"
38
+
39
+
40
+ def get_device(config):
41
+ if torch.cuda.is_available():
42
+ return 'cuda:' + str(config['cuda_device'])
43
+ return 'cpu'
44
+
45
+
46
+ def load_backbone(config):
47
+ if config['model'] != 'MERT':
48
+ raise ValueError(f"Unsupported model '{config['model']}'. This project builds a MERT quality model.")
49
+ return AutoModel.from_pretrained(MERT_MODEL_NAME, trust_remote_code=True, use_safetensors=True)
50
+
51
+
52
+ def inject_lora(module, target_names=("q_proj", "v_proj"), r=8, alpha=16, dropout=0.05):
53
+ """Wrap named Linear layers with LoRA adapters."""
54
+ for name, child in module.named_children():
55
+ if isinstance(child, torch.nn.Linear) and name in target_names:
56
+ module._modules[name] = LoRALinear(
57
+ base_layer=child,
58
+ r=r,
59
+ lora_alpha=alpha,
60
+ lora_dropout=dropout,
61
+ adapter_name=name,
62
+ )
63
+ else:
64
+ inject_lora(child, target_names=target_names, r=r, alpha=alpha, dropout=dropout)
65
+
66
+
67
+ def configure_finetune(backbone, config):
68
+ if config['freeze_feature_extractor']:
69
+ for param in backbone.feature_extractor.parameters():
70
+ param.requires_grad = False
71
+ elif config['freeze_all']:
72
+ for param in backbone.parameters():
73
+ param.requires_grad = False
74
+ elif config['LoRA']:
75
+ for param in backbone.parameters():
76
+ param.requires_grad = False
77
+ inject_lora(
78
+ backbone.encoder,
79
+ target_names=config['LoRA_target'],
80
+ r=config['LoRA_r'],
81
+ alpha=config['LoRA_alpha'],
82
+ dropout=config['LoRA_dropout'],
83
+ )
84
+ for name, param in backbone.encoder.named_parameters():
85
+ if "lora" in name:
86
+ param.requires_grad = True
87
+
88
+
89
+ def build_quality_model(config, backbone):
90
+ return MERT4AudioQuality(backbone, config['ssl_out_dim'], config['emb_dim'])
91
+
92
+
93
+ def lora_settings_for_checkpoint(config):
94
+ """LoRA hyperparameters from the eval config, overridden by the checkpoint's own config.yaml."""
95
+ settings = {
96
+ 'LoRA': bool(config.get('LoRA', False)),
97
+ 'LoRA_target': config.get('LoRA_target', ['q_proj', 'v_proj']),
98
+ 'LoRA_r': config.get('LoRA_r', 8),
99
+ 'LoRA_alpha': config.get('LoRA_alpha', 16),
100
+ 'LoRA_dropout': config.get('LoRA_dropout', 0.0),
101
+ }
102
+ sibling = os.path.join(os.path.dirname(config['model_path']), 'config.yaml')
103
+ if not os.path.isfile(sibling):
104
+ return settings
105
+ with open(sibling) as file:
106
+ saved = yaml.load(file, Loader=yaml.FullLoader) or {}
107
+ if not saved.get('LoRA'):
108
+ return settings
109
+ settings['LoRA'] = True
110
+ for key in ('LoRA_target', 'LoRA_r', 'LoRA_alpha', 'LoRA_dropout'):
111
+ if key in saved:
112
+ settings[key] = saved[key]
113
+ return settings
114
+
115
+
116
+ def apply_checkpoint_lora(backbone, config):
117
+ settings = lora_settings_for_checkpoint(config)
118
+ if not settings['LoRA']:
119
+ return
120
+ inject_lora(
121
+ backbone.encoder,
122
+ target_names=tuple(settings['LoRA_target']),
123
+ r=settings['LoRA_r'],
124
+ alpha=settings['LoRA_alpha'],
125
+ dropout=settings['LoRA_dropout'],
126
+ )
127
+
128
+
129
+ def load_state_dict_checked(module, path):
130
+ state = torch.load(path, map_location='cpu', weights_only=True)
131
+ missing, unexpected = module.load_state_dict(state, strict=False)
132
+ if missing or unexpected:
133
+ raise RuntimeError(
134
+ f"Checkpoint {path} does not match the model "
135
+ f"({len(missing)} missing, {len(unexpected)} unexpected). "
136
+ f"First missing: {missing[:5]}. First unexpected: {unexpected[:5]}."
137
+ )
138
+
139
+
140
+ class Training:
141
+ def __init__(self, config):
142
+ self.config = config
143
+ self.DEVICE = get_device(config)
144
+
145
+ backbone = load_backbone(config)
146
+ configure_finetune(backbone, config)
147
+ self.model = build_quality_model(config, backbone).to(self.DEVICE)
148
+
149
+ trainable = sum(p.numel() for p in self.model.parameters() if p.requires_grad)
150
+ total = sum(p.numel() for p in self.model.parameters())
151
+ print(f"Trainable params: {trainable:,} / {total:,} ({trainable / total:.2%})")
152
+
153
+ self.train_set = WSDataset(config, data_mode='train_df')
154
+ self.train_loader = DataLoader(
155
+ self.train_set,
156
+ batch_size=config['train_bs'],
157
+ shuffle=True,
158
+ drop_last=True,
159
+ num_workers=config['num_workers'],
160
+ collate_fn=self.train_set.collate_fn,
161
+ )
162
+
163
+ self.valid_set = WSDataset(config, data_mode='valid_df')
164
+ self.valid_loader = DataLoader(
165
+ self.valid_set,
166
+ batch_size=config['val_bs'],
167
+ shuffle=False,
168
+ drop_last=True,
169
+ num_workers=config['num_workers'],
170
+ collate_fn=self.valid_set.collate_fn,
171
+ )
172
+
173
+ self.criterion = RnC_Loss(config)
174
+ self.optim = self._build_optimizer()
175
+ self.lr_scheduler = lr_scheduler.ExponentialLR(self.optim, gamma=config['lr_decay_factor'])
176
+
177
+ def _build_optimizer(self):
178
+ if not self.config['LoRA']:
179
+ return torch.optim.Adam(self.model.parameters(), lr=self.config['lr'])
180
+
181
+ lora_params, base_params = [], []
182
+ for name, param in self.model.named_parameters():
183
+ if not param.requires_grad:
184
+ continue
185
+ (lora_params if "lora" in name else base_params).append(param)
186
+ return torch.optim.AdamW(
187
+ [
188
+ {"params": base_params, "lr": self.config['base_lr']},
189
+ {"params": lora_params, "lr": self.config['lora_lr']},
190
+ ],
191
+ weight_decay=self.config['weight_decay'],
192
+ )
193
+
194
+ def _run_epoch(self, dataloader, train):
195
+ self.model.train(train)
196
+ total_loss = 0.0
197
+ with torch.enable_grad() if train else torch.no_grad():
198
+ for wavs, labels, sub_labels, bitrate_labels in tqdm(dataloader):
199
+ wavs = wavs.to(self.DEVICE)
200
+ labels = labels.to(self.DEVICE)
201
+ sub_labels = sub_labels.to(self.DEVICE)
202
+ bitrate_labels = bitrate_labels.to(self.DEVICE)
203
+ loss = self.criterion(self.model(wavs), labels, sub_labels, bitrate_labels)
204
+ if train:
205
+ self.optim.zero_grad()
206
+ loss.backward()
207
+ self.optim.step()
208
+ total_loss += loss.item()
209
+ return total_loss / len(dataloader)
210
+
211
+ def training_loop(self):
212
+ dt_string = datetime.now().strftime("%d-%m-%Y_%H-%M-%S")
213
+ self.PATH_DIR = os.path.join('out-models', self.config['out_dir'], dt_string)
214
+ os.makedirs(self.PATH_DIR, exist_ok=True)
215
+ with open(os.path.join(self.PATH_DIR, 'config.yaml'), 'w') as file:
216
+ yaml.dump(self.config, file)
217
+
218
+ best_valid_loss = np.inf
219
+ counter = 0
220
+ for epoch in range(self.config['num_epochs']):
221
+ train_loss = self._run_epoch(self.train_loader, train=True)
222
+ valid_loss = self._run_epoch(self.valid_loader, train=False)
223
+
224
+ if valid_loss < best_valid_loss:
225
+ torch.save(self.model.state_dict(), os.path.join(self.PATH_DIR, 'best_model.pt'))
226
+ best_valid_loss = valid_loss
227
+ print("Saved Weights Success")
228
+ counter = 0
229
+ else:
230
+ counter += 1
231
+
232
+ if (counter + 1) % self.config['lr_decay_step'] == 0:
233
+ self.lr_scheduler.step()
234
+
235
+ print(f"COUNTER: {counter}/{self.config['patience']}")
236
+ print(f'LR: {self.lr_scheduler.get_last_lr()}')
237
+ if counter > self.config['patience']:
238
+ print('Stop training, counter greater than patience')
239
+ break
240
+
241
+ print(f"EPOCHS: {epoch + 1} train_loss : {train_loss}")
242
+ print(f"EPOCHS: {epoch + 1} valid_loss : {valid_loss}")
243
+ print('\n')
244
+
245
+
246
+ class Evaluation:
247
+ def __init__(self, config):
248
+ self.config = config
249
+ self.DEVICE = get_device(config)
250
+ backbone = load_backbone(config)
251
+ apply_checkpoint_lora(backbone, config)
252
+ self.model = build_quality_model(config, backbone)
253
+ load_state_dict_checked(self.model, config['model_path'])
254
+ self.model.to(self.DEVICE)
255
+
256
+ if config['mapping'] == 'nn':
257
+ self.mapping_model = DistanceToScoreNet()
258
+ load_state_dict_checked(self.mapping_model, config['mapping_model_path'])
259
+ self.mapping_model.to(self.DEVICE)
260
+
261
+ def eval_embedding_full_ref(self):
262
+ df = pd.read_csv(self.config['test_files_csv'])
263
+ root = self.config['test_root_wav']
264
+ ref_embeddings = self.get_embeddings_csv(df['filepath_ref'], root).set_index('filepath_ref')
265
+ test_embeddings = self.get_embeddings_csv(df['filepath_deg'], root).set_index('filepath_deg')
266
+ self._save_distance_results(df, np.diag(cdist(test_embeddings, ref_embeddings)))
267
+
268
+ def eval_embedding_non_matching_ref(self):
269
+ df = pd.read_csv(self.config['test_files_csv'])
270
+ ref_root = self.config['non_matching_Mref_root']
271
+ ref_paths = [
272
+ os.path.relpath(path, ref_root)
273
+ for path in glob.glob(os.path.join(ref_root, "**", "*.wav"), recursive=True)
274
+ ]
275
+ ref_embeddings = self.get_embeddings_csv(pd.Series(ref_paths, name='filepath_ref'), ref_root).iloc[:, 1:].values
276
+ test_embeddings = self.get_embeddings_csv(df['filepath_deg'], self.config['test_root_wav']).iloc[:, 1:].values
277
+ self._save_distance_results(df, np.mean(cdist(test_embeddings, ref_embeddings), axis=1))
278
+
279
+ def get_embeddings_csv(self, file_names, root=None):
280
+ embeddings = []
281
+ self.model.eval()
282
+ with torch.no_grad():
283
+ for filename in tqdm(np.array(file_names)):
284
+ filepath = os.path.join(root, filename) if root else filename
285
+ wav = load_processing(
286
+ filepath,
287
+ target_sr=self.config['sampling_rate'],
288
+ trim=False,
289
+ wav_norm=False,
290
+ ).to(self.DEVICE).unsqueeze(0)
291
+ embeddings.append(self.model(wav).squeeze().cpu().detach().numpy())
292
+
293
+ embedding_frame = pd.DataFrame(np.array(embeddings))
294
+ return pd.concat([file_names.reset_index(), embedding_frame], axis=1).drop('index', axis=1)
295
+
296
+ def _map_distances(self, distances, df):
297
+ mapping = self.config['mapping']
298
+ if mapping == 'polynomial':
299
+ return self.polynomial_mapping(distances, df['SG'])
300
+ if mapping == 'spline':
301
+ return self.spline_mapping(distances, df['SG'])
302
+ if mapping == 'nn':
303
+ return self.neural_network_mapping(distances)
304
+ return None
305
+
306
+ def _save_distance_results(self, df, distances):
307
+ os.makedirs(self.config['emb_out_dir'], exist_ok=True)
308
+ mapped_result = self._map_distances(distances, df)
309
+ out_path = os.path.join(self.config['emb_out_dir'], self.config['emb_out_file'])
310
+
311
+ if 'SG' not in df.columns or len(df) <= 1:
312
+ if self.config['mapping'] == 'nn':
313
+ df_dist = pd.DataFrame({'filepath_deg': df['filepath_deg'], 'l2': distances, 'mapped': mapped_result})
314
+ else:
315
+ print('Subjective scores are absent or too few samples; cannot be mapped with Polynomial or Spline')
316
+ df_dist = pd.DataFrame({'filepath_deg': df['filepath_deg'], 'l2': distances})
317
+ df_dist.to_csv(out_path, index=False)
318
+ return
319
+
320
+ df_dist = pd.DataFrame({
321
+ 'filepath_deg': df['filepath_deg'],
322
+ 'l2': distances,
323
+ 'mapped': mapped_result,
324
+ 'SG': df['SG'],
325
+ })
326
+ srcc, _ = spearmanr(mapped_result, df['SG'].values)
327
+ pcc, _ = pearsonr(mapped_result, df['SG'].values)
328
+ print(f"Evaluation → SRCC: {srcc:.3f}, PCC: {pcc:.3f}")
329
+
330
+ gt_list = pd.to_numeric(df_dist['SG'], errors='coerce').astype(float)
331
+ pd_list = pd.to_numeric(df_dist['mapped'], errors='coerce').astype(float)
332
+ slope, intercept, _, _, _ = linregress(gt_list, pd_list)
333
+ line_x = np.linspace(0, 100, 100)
334
+ line_y = slope * line_x + intercept
335
+
336
+ plt.figure(figsize=(8, 8))
337
+ plt.scatter(gt_list, pd_list, alpha=0.7, edgecolors='k', s=40, label='Samples')
338
+ plt.plot(line_x, line_y, 'r--', linewidth=2.5, label='Linear Fit')
339
+ plt.xlabel('Ground Truth', fontsize=16)
340
+ plt.ylabel('Mapped Results', fontsize=16)
341
+ plt.xlim(0, 100)
342
+ plt.ylim(0, 100)
343
+ plt.grid(True, linestyle='--', alpha=0.5)
344
+ plt.legend(fontsize=14)
345
+ plt.text(
346
+ 0.05, 0.9, f"SRCC = {srcc:.3f}\nPCC = {pcc:.3f}",
347
+ transform=plt.gca().transAxes, fontsize=14, verticalalignment='top',
348
+ bbox=dict(boxstyle='round', facecolor='white', alpha=0.7),
349
+ )
350
+ plt.tight_layout()
351
+ plt.savefig(os.path.join(self.config['emb_out_dir'], self.config['plot_out_file']), dpi=300)
352
+ plt.close()
353
+ df_dist.to_csv(out_path, index=False)
354
+
355
+ @staticmethod
356
+ def polynomial_mapping(pred_values, gt_values):
357
+ def order_three(x, a, b, c, d):
358
+ return a * x + b * x ** 2 + c * x ** 3 + d
359
+
360
+ popt, _ = curve_fit(order_three, pred_values, gt_values)
361
+ return order_three(pred_values, *popt)
362
+
363
+ @staticmethod
364
+ def spline_mapping(pred_values, gt_values):
365
+ x_data = np.array(pred_values)
366
+ y_data = np.array(gt_values)
367
+ sort_idx = np.argsort(x_data)
368
+ x_sorted = x_data[sort_idx]
369
+ y_sorted = y_data[sort_idx]
370
+
371
+ unique_x, inv_idx = np.unique(x_sorted, return_inverse=True)
372
+ y_mean = np.array([y_sorted[inv_idx == j].mean() for j in range(len(unique_x))])
373
+ x_knots = np.linspace(unique_x.min(), unique_x.max(), 5)[1:-1]
374
+ spline = LSQUnivariateSpline(unique_x, y_mean, t=x_knots, k=2)
375
+ return spline(x_data)
376
+
377
+ def neural_network_mapping(self, pred_values):
378
+ self.mapping_model.eval()
379
+ mapped = []
380
+ with torch.no_grad():
381
+ for x in tqdm(pred_values):
382
+ y_pred = self.mapping_model(torch.tensor([[x]], dtype=torch.float32, device=self.DEVICE))
383
+ mapped.append(y_pred.item())
384
+ return np.array(mapped)
@@ -0,0 +1,143 @@
1
+ Metadata-Version: 2.4
2
+ Name: DeePAQ
3
+ Version: 0.1.0
4
+ Summary: MERT-based audio quality embeddings and reference evaluation
5
+ Author: Guanxin Jiang, Fraunhofer IIS
6
+ License-Expression: MIT
7
+ Project-URL: Homepage, https://git.audiolabs.uni-erlangen.de/guanxin.jiang/audio-quality-assessment-dnn
8
+ Keywords: audio,quality-assessment,MERT
9
+ Classifier: Programming Language :: Python :: 3
10
+ Classifier: Programming Language :: Python :: 3.10
11
+ Classifier: Operating System :: OS Independent
12
+ Requires-Python: >=3.10
13
+ Description-Content-Type: text/markdown
14
+ License-File: MIT_License.txt
15
+ Requires-Dist: numpy
16
+ Requires-Dist: pandas
17
+ Requires-Dist: scipy
18
+ Requires-Dist: matplotlib
19
+ Requires-Dist: seaborn
20
+ Requires-Dist: tqdm
21
+ Requires-Dist: pyyaml
22
+ Requires-Dist: click
23
+ Requires-Dist: torch>=2.5.1
24
+ Requires-Dist: torchaudio>=2.5.1
25
+ Requires-Dist: transformers==4.57.1
26
+ Requires-Dist: peft==0.17.1
27
+ Requires-Dist: nnAudio==0.3.4
28
+ Dynamic: license-file
29
+
30
+ # DeePAQ
31
+
32
+ DeePAQ uses pre-trained MERT to extract embeddings of size 256. The provided `training_mapping.py` maps embeddings to a MUSHRA-like scale (0–100).
33
+
34
+ ## Table of Contents
35
+
36
+ * [Installation](#installation)
37
+ * [Dataset](#dataset)
38
+ * [Usage](#usage)
39
+ * [Training](#training)
40
+ * [Evaluation](#evaluation)
41
+ * [Contributing](#contributing)
42
+ * [License](#license)
43
+
44
+ ## Installation
45
+ DeePAQ is built and tested on PyTorch 2.5.1, TorchVision 0.20.1, TorchAudio 2.5.1 and CUDA 12.1 runtime
46
+ It can be either installed with pip or git.
47
+
48
+ ### pip
49
+ Checkpoints are not part of the PyPI package. Download the DeePAQ checkpoints, and the neural mapping checkpoint if you use `mapping: nn`, and point `model_path` / `mapping_model_path` at those files.
50
+
51
+ ```bash
52
+ pip install DeePAQ
53
+ ```
54
+
55
+ From a local build of this repository:
56
+
57
+ ```bash
58
+ pip install dist/deepaq-0.1.0-py3-none-any.whl
59
+ ```
60
+
61
+ ### git
62
+ ```bash
63
+ git clone https://git.audiolabs.uni-erlangen.de/guanxin.jiang/audio-quality-assessment-dnn.git
64
+ cd audio-quality-assessment-dnn
65
+ git checkout DeePAQ
66
+ conda env create -f environment.yml
67
+ conda activate DeePAQ-env
68
+ ```
69
+
70
+ ## Dataset
71
+
72
+ For evaluation the project requires signals under test, matched reference signals, and corresponding subjective scores in CSV format. Each CSV should include columns:
73
+
74
+ * `filepath_deg` — path to the degraded/test signal
75
+ * `filepath_ref` — (optional) path to the matched reference signal
76
+ * `SG` — (optional) subjective rating. If SG column exists in your csv file, then the script will automatically detect it and then calculate the overall Spearman and Pearson correlation between the mapped results and subjective ratings
77
+
78
+ Place your dataset files in the `data/` folder.
79
+
80
+ ## Usage
81
+
82
+ ### Training
83
+
84
+ Run the main script with your training config:
85
+
86
+ ```bash
87
+ python main.py --config_file src/config/train1_1.yaml
88
+ ```
89
+
90
+ ### Evaluation
91
+ #### Extract Embeddings
92
+ If you pip install deepaq
93
+
94
+ ```python
95
+ import deepaq
96
+ wav_path='/path/test.wav'
97
+ checkpoint='/path/best_model.pt'
98
+ embs = deepaq.compute_embedding(wav_path, checkpoint, model_type='MERT', mapping_type='polynomial')
99
+ ```
100
+
101
+ #### Full Reference
102
+
103
+ ```bash
104
+ python main.py --config_file src/config/fr_eval.yaml
105
+ ```
106
+ or if you pip install deepaq
107
+ ```bash
108
+ deepaq --config_file src/config/fr_eval.yaml
109
+ ```
110
+
111
+ In fr_eval.yaml config file
112
+ * `model` — foundation model
113
+ * `model_path` — checkpoint has been provided in out-models folder. For full reference, 2 checkpoints are provided, which are MERT-FR-2f and MERT-FR-best
114
+ * `mapping` - choose the mapping functions: polynomial uses 3rd order polynomial mapping; spline uses quadratic B-spline (degree 2, order 3) with three internal knots; nn uses a neural network trained to map Euclidean distance to MUSHRA scale.
115
+ * `mapping_model_path`: ./out-models/Dist2SG_Mapping/best_model.pt (This is optional if polynomial or spline is used for mapping)
116
+ * `test_files_csv` — csv file containing the data for testing
117
+ * `test_root_wav` — folder where your test data stored
118
+ * `emb_out_dir` - folder where you want to save your evalution results
119
+ * `emb_out_file` — csv file saves the evaluation results
120
+ * `plot_out_file` — png file saves scatterplot of the evaluation results
121
+
122
+ #### Non-matching Reference
123
+
124
+ ```bash
125
+ python main.py --config_file src/config/nmr_eval.yaml
126
+ ```
127
+ In nmr_eval.yaml config file
128
+ * `model` — foundation model
129
+ * `model_path` — checkpoint has been provided in out-models folder.
130
+ * `mapping` - choose the mapping functions: polynomial uses 3rd order polynomial mapping; spline uses quadratic B-spline (degree 2, order 3) with three internal knots; nn uses a neural network trained to map Euclidean distance to MUSHRA scale.
131
+ * `mapping_model_path`: ./out-models/Dist2SG_Mapping/best_model.pt (This is optional if polynomial or spline is used for mapping)
132
+ * `test_files_csv` — csv file containing the data for testing
133
+ * `test_root_wav` — folder where your test data stored
134
+ * `emb_out_dir` - folder where you want to save your evalution results
135
+ * `emb_out_file` — csv file saves the evaluation results
136
+ * `plot_out_file` — png file saves scatterplot of the evaluation results
137
+ * `non_matching_Mref_root` — folder where contains the non-matching reference with music content
138
+ * `non_matching_Sref_root` — folder where contains the non-matching reference with speech content
139
+
140
+ ## Notation
141
+ model wav2vec is for development. If users want to use DeePAQ for inference, wav2vec could be skipped; otherwise need to install fairseq for wav2vec
142
+
143
+
@@ -0,0 +1,16 @@
1
+ deepaq/__init__.py,sha256=lz9okMrbXELG4RViyKdCcZupQSdKbZ6KOTCqHruuEq8,1362
2
+ deepaq/cli.py,sha256=M47jWrW8Q8OIAsX9IcInlVKlrW_Toj-xGajeZx9JOWI,749
3
+ deepaq/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
4
+ deepaq/data/dataloader.py,sha256=iwSRWJQ0hcHxkRCXzIGHVFAm66ELcy8JAmD-RDwaj-g,2841
5
+ deepaq/losses/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
6
+ deepaq/losses/loss.py,sha256=YPjPhx81B3Y06Vg-_8oussHiGEMHOHwP7AZUYrd7I0c,2279
7
+ deepaq/networks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
8
+ deepaq/networks/network.py,sha256=srsC4FpZvm_7vIXey1NvBhdnvtObftUSn_jXNGxXXNI,1130
9
+ deepaq/train/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
10
+ deepaq/train/training.py,sha256=Igw3DXg2iRYXtVeBZR19Efs1whDclBRTtA5WY9s9wH4,15334
11
+ deepaq-0.1.0.dist-info/licenses/MIT_License.txt,sha256=O9n2NwvipHY4Mb01jqd7XUEaHajUfzQlKSR2mIX4D88,1084
12
+ deepaq-0.1.0.dist-info/METADATA,sha256=AtoKqItu8YUVjcWk8_yF-E0IUZIBUU8O-kiiX6rljuU,5383
13
+ deepaq-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
14
+ deepaq-0.1.0.dist-info/entry_points.txt,sha256=WitM2hFPpYZ6NSv6Et-XD7pcN3ACAdh4dGV_0Uqg-AE,43
15
+ deepaq-0.1.0.dist-info/top_level.txt,sha256=BGFspTm3rPFT1jiX1L9LI-vnpLXLs2mvOfpsPt1xLdY,7
16
+ deepaq-0.1.0.dist-info/RECORD,,
@@ -0,0 +1,5 @@
1
+ Wheel-Version: 1.0
2
+ Generator: setuptools (84.2.1)
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
5
+
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ deepaq = deepaq.cli:main
@@ -0,0 +1,9 @@
1
+ MIT License
2
+
3
+ Copyright 2026 Fraunhofer IIS
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
6
+
7
+ The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
8
+
9
+ THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1 @@
1
+ deepaq