describealign 1.1.2__py3-none-any.whl → 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {describealign-1.1.2.dist-info → describealign-2.0.0.dist-info}/METADATA +8 -9
- describealign-2.0.0.dist-info/RECORD +7 -0
- {describealign-1.1.2.dist-info → describealign-2.0.0.dist-info}/WHEEL +1 -1
- describealign.py +1494 -1025
- describealign-1.1.2.dist-info/RECORD +0 -7
- {describealign-1.1.2.dist-info → describealign-2.0.0.dist-info}/entry_points.txt +0 -0
- {describealign-1.1.2.dist-info → describealign-2.0.0.dist-info}/licenses/LICENSE +0 -0
- {describealign-1.1.2.dist-info → describealign-2.0.0.dist-info}/top_level.txt +0 -0
describealign.py
CHANGED
|
@@ -2,10 +2,6 @@
|
|
|
2
2
|
# input: video or folder of videos and an audio file or folder of audio files
|
|
3
3
|
# output: videos in a folder "videos_with_ad", with aligned segments of the audio replaced
|
|
4
4
|
# this script aligns the new audio to the video using the video's old audio
|
|
5
|
-
# first, the video's sound and the audio file are both converted to spectrograms
|
|
6
|
-
# second, the two spectrograms are roughly aligned by finding their longest common subsequence
|
|
7
|
-
# third, the rough alignment is denoised through L1-Minimization
|
|
8
|
-
# fourth, the spectrogram alignments determine where the new audio replaces the old
|
|
9
5
|
|
|
10
6
|
'''
|
|
11
7
|
Copyright (C) 2023 Julian Brown
|
|
@@ -28,26 +24,14 @@ VIDEO_EXTENSIONS = set(['mp4', 'mkv', 'avi', 'mov', 'webm', 'm4v', 'flv', 'vob']
|
|
|
28
24
|
AUDIO_EXTENSIONS = set(['mp3', 'm4a', 'opus', 'wav', 'aac', 'flac', 'ac3', 'mka'])
|
|
29
25
|
PLOT_ALIGNMENT_TO_FILE = True
|
|
30
26
|
|
|
31
|
-
|
|
32
|
-
|
|
27
|
+
TIMESTEPS_PER_SECOND = 10 # factors must be subset of (2, 3, 5, 7)
|
|
28
|
+
TIMESTEP_SIZE_SECONDS = 1. / TIMESTEPS_PER_SECOND
|
|
33
29
|
AUDIO_SAMPLE_RATE = 44100
|
|
34
|
-
|
|
35
|
-
DITHER_PERIOD_STEPS = 60
|
|
36
|
-
MIN_CORR_FOR_TOKEN_MATCH = .6
|
|
37
|
-
GAP_START_COST = 1.0
|
|
38
|
-
GAP_EXTEND_COST = -.01
|
|
39
|
-
GAP_EXTEND_DIAG_BONUS = -.01
|
|
40
|
-
SKIP_MATCH_COST = .1
|
|
30
|
+
DITHER_PERIOD_STEPS = 10
|
|
41
31
|
MAX_RATE_RATIO_DIFF_ALIGN = .1
|
|
42
|
-
PREF_CUT_AT_GAPS_FACTOR = 5
|
|
43
32
|
MIN_DURATION_TO_REPLACE_SECONDS = 2
|
|
44
|
-
MIN_START_END_SYNC_TIME_SECONDS = 2
|
|
45
|
-
MAX_START_END_SYNC_ERR_SECONDS = .2
|
|
46
|
-
MAX_RATE_RATIO_DIFF_BOOST = .003
|
|
47
|
-
MIN_DESC_DURATION = .5
|
|
48
|
-
MAX_GAP_IN_DESC_SEC = 1.5
|
|
49
33
|
JUST_NOTICEABLE_DIFF_IN_FREQ_RATIO = .005
|
|
50
|
-
|
|
34
|
+
MIN_STRETCH_OFFSET = 30
|
|
51
35
|
|
|
52
36
|
if PLOT_ALIGNMENT_TO_FILE:
|
|
53
37
|
import matplotlib.pyplot as plt
|
|
@@ -64,42 +48,41 @@ import numpy as np
|
|
|
64
48
|
import ffmpeg
|
|
65
49
|
import platformdirs
|
|
66
50
|
import static_ffmpeg
|
|
67
|
-
import python_speech_features as psf
|
|
68
51
|
import scipy.signal
|
|
69
52
|
import scipy.optimize
|
|
70
53
|
import scipy.interpolate
|
|
71
|
-
import scipy.ndimage as nd
|
|
72
54
|
import scipy.sparse
|
|
73
|
-
import pytsmod
|
|
74
55
|
import configparser
|
|
75
56
|
import traceback
|
|
76
57
|
import multiprocessing
|
|
77
58
|
import platform
|
|
59
|
+
import natsort
|
|
60
|
+
from collections import defaultdict
|
|
61
|
+
from sortedcontainers import SortedList
|
|
62
|
+
import hashlib
|
|
63
|
+
|
|
64
|
+
try:
|
|
65
|
+
import wx
|
|
66
|
+
gui_font = (11, wx.FONTFAMILY_SWISS, wx.FONTSTYLE_NORMAL, wx.FONTWEIGHT_NORMAL, False, "Arial")
|
|
67
|
+
except ImportError:
|
|
68
|
+
wx = None
|
|
69
|
+
|
|
70
|
+
gui_update_interval_ms = 100
|
|
71
|
+
gui_background_color_dark = (28, 30, 35)
|
|
72
|
+
gui_background_color_light = (170, 182, 211)
|
|
78
73
|
|
|
79
74
|
IS_RUNNING_WINDOWS = platform.system() == 'Windows'
|
|
80
75
|
if IS_RUNNING_WINDOWS:
|
|
81
|
-
import PySimpleGUIWx as sg
|
|
82
76
|
default_output_dir = 'videos_with_ad'
|
|
83
77
|
default_alignment_dir = 'alignment_plots'
|
|
84
78
|
else:
|
|
85
|
-
import PySimpleGUIQt as sg
|
|
86
79
|
default_output_dir = os.path.expanduser('~') + '/videos_with_ad'
|
|
87
80
|
default_alignment_dir = os.path.expanduser('~') + '/alignment_plots'
|
|
88
81
|
|
|
89
|
-
def
|
|
90
|
-
if func:
|
|
91
|
-
func(text)
|
|
92
|
-
print(text)
|
|
93
|
-
|
|
94
|
-
def throw_runtime_error(text, func=None):
|
|
95
|
-
if func:
|
|
96
|
-
func(text)
|
|
97
|
-
raise RuntimeError(text)
|
|
98
|
-
|
|
99
|
-
def ensure_folders_exist(dirs, display_func=None):
|
|
82
|
+
def ensure_folders_exist(dirs):
|
|
100
83
|
for dir in dirs:
|
|
101
84
|
if not os.path.isdir(dir):
|
|
102
|
-
|
|
85
|
+
print(f"Directory not found, creating it: {dir}")
|
|
103
86
|
os.makedirs(dir)
|
|
104
87
|
|
|
105
88
|
def get_sorted_filenames(path, extensions, alt_extensions=set([])):
|
|
@@ -127,346 +110,59 @@ def get_sorted_filenames(path, extensions, alt_extensions=set([])):
|
|
|
127
110
|
"Or maybe you need to add a new extension to this script's regex?",
|
|
128
111
|
f"valid extensions for this input are:\n {extensions}"]
|
|
129
112
|
raise RuntimeError("\n".join(error_msg))
|
|
130
|
-
files =
|
|
131
|
-
|
|
132
|
-
return files,
|
|
133
|
-
|
|
134
|
-
# read audio from file with ffmpeg and convert to numpy array
|
|
135
|
-
def parse_audio_from_file(media_file):
|
|
136
|
-
media_stream, _ = (ffmpeg
|
|
137
|
-
.input(media_file)
|
|
138
|
-
.output('-', format='s16le', acodec='pcm_s16le', ac=2, ar=AUDIO_SAMPLE_RATE, loglevel='fatal')
|
|
139
|
-
.run(capture_stdout=True, cmd=get_ffmpeg())
|
|
140
|
-
)
|
|
141
|
-
media_arr = np.frombuffer(media_stream, np.int16).astype(np.float32).reshape((-1,2)).T
|
|
142
|
-
return media_arr
|
|
113
|
+
files = natsort.os_sorted(files)
|
|
114
|
+
has_alt_extensions = [0 if os.path.splitext(file)[1][1:] in extensions else 1 for file in files]
|
|
115
|
+
return files, has_alt_extensions
|
|
143
116
|
|
|
144
|
-
#
|
|
145
|
-
def
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
winstep=TIMESTEP_SIZE_SECONDS * rate,
|
|
154
|
-
numcep=MEL_COEFFS_PER_TIMESTEP,
|
|
155
|
-
nfilt=MEL_COEFFS_PER_TIMESTEP * 2,
|
|
156
|
-
nfft=fft_size_samples,
|
|
157
|
-
winfunc=scipy.signal.windows.hann)
|
|
158
|
-
num_timesteps = max(1, ((media_arr.shape[1] - window_size_samples - 1) // step_size_samples) + 2)
|
|
159
|
-
media_spec = np.zeros((num_timesteps, MEL_COEFFS_PER_TIMESTEP))
|
|
160
|
-
chunk_size = 1000
|
|
161
|
-
for chunk_index in np.arange(0, num_timesteps, chunk_size):
|
|
162
|
-
chunk_bounds_samples = ((chunk_index ) * step_size_samples,
|
|
163
|
-
(chunk_index + chunk_size - 1) * step_size_samples + window_size_samples)
|
|
164
|
-
media_spec[chunk_index:chunk_index+chunk_size] = get_mfcc(media_arr[:,slice(*chunk_bounds_samples)])
|
|
165
|
-
'''
|
|
166
|
-
# alternate python library's MFC implementation
|
|
167
|
-
import librosa
|
|
168
|
-
media_spec = librosa.feature.mfcc(y=np.mean(media_arr, axis=0),
|
|
169
|
-
sr=AUDIO_SAMPLE_RATE,
|
|
170
|
-
n_mfcc=MEL_COEFFS_PER_TIMESTEP,
|
|
171
|
-
lifter=22,
|
|
172
|
-
n_fft=fft_size_samples,
|
|
173
|
-
hop_length=step_size_samples,
|
|
174
|
-
win_length=window_size_samples,
|
|
175
|
-
window=scipy.signal.windows.hann).T
|
|
176
|
-
num_timesteps = media_spec.shape[0]
|
|
177
|
-
'''
|
|
178
|
-
timings_samples = window_size_samples/2. + step_size_samples * np.arange(num_timesteps)
|
|
179
|
-
timings_seconds = timings_samples / AUDIO_SAMPLE_RATE
|
|
180
|
-
return media_spec, timings_seconds
|
|
181
|
-
|
|
182
|
-
# same as tokenize_audio, but dithering the MFC window timings
|
|
183
|
-
# this allows for finer alignment by ameliorating discretization error
|
|
184
|
-
def tokenize_audio_dither(media_arr, slow_timings):
|
|
185
|
-
# choose a relative step size slightly less than 1 to ameliorate quantization error
|
|
186
|
-
# maximize alignment accuracy by using least approximable number with desired period
|
|
187
|
-
# this is the continued fraction [0;1,N-2,1,1,1,...], where the trailing ones give phi
|
|
188
|
-
fast_rate = 1. / (1 + 1. / (DITHER_PERIOD_STEPS - 2 + (np.sqrt(5) + 1) / 2.))
|
|
189
|
-
fast_spec, fast_timings = tokenize_audio(media_arr, fast_rate)
|
|
190
|
-
|
|
191
|
-
# prevent drift in difficult to align segments (e.g. describer speaking or quiet/droning segments)
|
|
192
|
-
# by approximately equalizing the number of tokens per unit time between dithered and undithered
|
|
193
|
-
# the dithered audio will have ~(1 + 1 / DITHER_PERIOD_STEPS) times as many tokens, so
|
|
194
|
-
# this can be accomplished by simply deleting a token every DITHER_PERIOD_STEPS tokens
|
|
195
|
-
fast_spec = np.delete(fast_spec, slice(DITHER_PERIOD_STEPS // 2, None, DITHER_PERIOD_STEPS), axis=0)
|
|
196
|
-
fast_timings = np.delete(fast_timings, slice(DITHER_PERIOD_STEPS // 2, None, DITHER_PERIOD_STEPS))
|
|
197
|
-
return fast_spec, fast_timings
|
|
198
|
-
|
|
199
|
-
# normalize along both time and frequency axes to allow comparing tokens by correlation
|
|
200
|
-
def normalize_spec(media_spec_raw, axes=(0,1)):
|
|
201
|
-
media_spec = media_spec_raw.copy()
|
|
202
|
-
for axis in axes:
|
|
203
|
-
norm_func = np.std if axis == 0 else np.linalg.norm
|
|
204
|
-
media_spec = media_spec - np.mean(media_spec, axis=axis, keepdims=True)
|
|
205
|
-
media_spec = media_spec/(norm_func(media_spec,axis=axis,keepdims=True)+1e-10)
|
|
206
|
-
return media_spec
|
|
207
|
-
|
|
208
|
-
# vectorized implementation of the Wagner–Fischer (Longest Common Subsequence) algorithm
|
|
209
|
-
# modified to include affine gap penalties and skip+match options (i.e. knight's moves)
|
|
210
|
-
# gaps are necessary when parts are cut out of the audio description (e.g. cut credits)
|
|
211
|
-
# or when the audio description includes a commercial break or an extra scene
|
|
212
|
-
# the skip+match option allows for micro-adjustments without eating the full gap penalty
|
|
213
|
-
# skip+match is primarily useful in maintaining alignment when the rates differ slightly
|
|
214
|
-
def rough_align(video_spec, audio_desc_spec, video_timings, audio_desc_timings):
|
|
215
|
-
pred_map = {0:lambda node: (0, node[1]-1, node[2]-1),
|
|
216
|
-
1:lambda node: (0, node[1]-2, node[2]-1),
|
|
217
|
-
2:lambda node: (0, node[1]-1, node[2]-2),
|
|
218
|
-
3:lambda node: (1, node[1]-1, node[2]-1),
|
|
219
|
-
4:lambda node: (0, node[1] , node[2] ),
|
|
220
|
-
5:lambda node: (1, node[1]-1, node[2] ),
|
|
221
|
-
6:lambda node: (1, node[1]-1, node[2]-1),
|
|
222
|
-
7:lambda node: (1, node[1] , node[2]-1)}
|
|
223
|
-
pred_matrix = np.zeros((2, audio_desc_spec.shape[0], video_spec.shape[0]), dtype=np.uint8)
|
|
224
|
-
pred_matrix[0,1:,:2] = 0
|
|
225
|
-
pred_matrix[1,1:,:2] = 4
|
|
226
|
-
pred_matrix[:,0,:2] = [0,5]
|
|
227
|
-
path_corrs_match = np.zeros((3, video_spec.shape[0]))
|
|
228
|
-
path_corrs_gap = np.zeros((3, video_spec.shape[0]))
|
|
229
|
-
corrs = np.zeros((3, video_spec.shape[0]))
|
|
230
|
-
corrs[:,:] = np.roll(np.dot(video_spec, audio_desc_spec[0]), 1)[None,:]
|
|
231
|
-
for i in range(audio_desc_spec.shape[0]):
|
|
232
|
-
i_mod = i % 3
|
|
233
|
-
match_pred_corrs = np.hstack([path_corrs_match[i_mod-1][1:-1][:,None],
|
|
234
|
-
path_corrs_match[i_mod-2][1:-1][:,None] - SKIP_MATCH_COST,
|
|
235
|
-
path_corrs_match[i_mod-1][0:-2][:,None] - SKIP_MATCH_COST,
|
|
236
|
-
path_corrs_gap[ i_mod-1][1:-1][:,None]])
|
|
237
|
-
pred_matrix[0][i][2:] = np.argmax(match_pred_corrs, axis=1)
|
|
238
|
-
path_corrs_match[i_mod][2:] = np.take_along_axis(match_pred_corrs, pred_matrix[0][i][2:,None], axis=1).T
|
|
239
|
-
corrs = np.roll(corrs, -1, axis=1)
|
|
240
|
-
corrs[(i_mod+1)%3,:] = np.roll(np.dot(video_spec, audio_desc_spec[min(audio_desc_spec.shape[0]-1,i+1)]), 1)
|
|
241
|
-
fisher_infos = (2 * corrs[i_mod] - corrs[i_mod-1] - corrs[(i_mod+1)%3]) / min(.2, TIMESTEP_SIZE_SECONDS)
|
|
242
|
-
fisher_infos[fisher_infos < 0] = 0
|
|
243
|
-
fisher_infos[fisher_infos > 10] = 10
|
|
244
|
-
row_corrs = np.maximum(0, corrs[i_mod][2:] - MIN_CORR_FOR_TOKEN_MATCH)
|
|
245
|
-
path_corrs_match[i_mod][2:] += row_corrs * (fisher_infos[2:] / 5)
|
|
246
|
-
gap_pred_corrs = np.hstack([path_corrs_match[i_mod][2: ][:,None] - GAP_START_COST,
|
|
247
|
-
path_corrs_gap[i_mod-1][2: ][:,None],
|
|
248
|
-
path_corrs_gap[i_mod-1][1:-1][:,None] - GAP_EXTEND_DIAG_BONUS - \
|
|
249
|
-
GAP_EXTEND_COST])
|
|
250
|
-
pred_matrix[1][i][2:] = np.argmax(gap_pred_corrs, axis=1)
|
|
251
|
-
path_corrs_gap_no_col_skip = np.take_along_axis(gap_pred_corrs, pred_matrix[1][i][2:,None], axis=1).flat
|
|
252
|
-
pred_matrix[1][i][2:] += 4
|
|
253
|
-
path_corrs_gap[i_mod][2:] = np.maximum.accumulate(path_corrs_gap_no_col_skip + \
|
|
254
|
-
GAP_EXTEND_COST * np.arange(video_spec.shape[0]-2)) - \
|
|
255
|
-
GAP_EXTEND_COST * np.arange(video_spec.shape[0]-2)
|
|
256
|
-
pred_matrix[1][i][2:][path_corrs_gap[i_mod][2:] > path_corrs_gap_no_col_skip] = 7
|
|
257
|
-
path_corrs_gap[i_mod][2:] -= GAP_EXTEND_COST
|
|
258
|
-
|
|
259
|
-
# reconstruct optimal path by following predecessors backwards through the table
|
|
260
|
-
end_node_layer = np.argmax([path_corrs_match[i_mod,-1],
|
|
261
|
-
path_corrs_gap[ i_mod,-1]])
|
|
262
|
-
cur_node = (end_node_layer, audio_desc_spec.shape[0]-1, video_spec.shape[0]-1)
|
|
263
|
-
get_predecessor = lambda node: pred_map[pred_matrix[node]](node)
|
|
264
|
-
path = []
|
|
265
|
-
visited = set()
|
|
266
|
-
while min(cur_node[1:]) >= 0:
|
|
267
|
-
cur_node, last_node = get_predecessor(cur_node), cur_node
|
|
268
|
-
# failsafe to prevent an infinite loop that should never happen anyways
|
|
269
|
-
if cur_node in visited:
|
|
270
|
-
break
|
|
271
|
-
visited.add(cur_node)
|
|
272
|
-
if last_node[0] == 0:
|
|
273
|
-
path.append(last_node[1:])
|
|
274
|
-
path = path[::-1]
|
|
275
|
-
|
|
276
|
-
# determine how much information this node gives about the alignment
|
|
277
|
-
# a larger double derivative means more precise timing information
|
|
278
|
-
# sudden noises give more timing information than droning sounds
|
|
279
|
-
def get_fisher_info(node):
|
|
280
|
-
i,j = node
|
|
281
|
-
if node[0] >= audio_desc_spec.shape[0]-1 or \
|
|
282
|
-
node[1] >= video_spec.shape[0]-1 or \
|
|
283
|
-
min(node) <= 0:
|
|
284
|
-
return 0
|
|
285
|
-
info = 2*np.dot(audio_desc_spec[i ],video_spec[j ]) - \
|
|
286
|
-
np.dot(audio_desc_spec[i-1],video_spec[j+1]) - \
|
|
287
|
-
np.dot(audio_desc_spec[i+1],video_spec[j-1])
|
|
288
|
-
info /= min(.2, TIMESTEP_SIZE_SECONDS)
|
|
289
|
-
return info
|
|
290
|
-
|
|
291
|
-
# the quality of a node combines the correlation of its tokens
|
|
292
|
-
# with how precisely the match is localized in time
|
|
293
|
-
def get_match_quality(node):
|
|
294
|
-
# correlations are between -1 and 1, as all tokens have unit norm
|
|
295
|
-
token_correlation = np.dot(audio_desc_spec[node[0]],video_spec[node[1]])
|
|
296
|
-
fisher_info = min(max(0, get_fisher_info(node)), 10)
|
|
297
|
-
return max(0, token_correlation - MIN_CORR_FOR_TOKEN_MATCH) * (fisher_info / 5)
|
|
298
|
-
|
|
299
|
-
# filter out low match quality nodes from LCS path
|
|
300
|
-
quals = [get_match_quality(node) for node in path]
|
|
301
|
-
if len(quals) == 0 or max(quals) <= 0:
|
|
302
|
-
raise RuntimeError("Rough alignment failed, are the input files mismatched?")
|
|
303
|
-
path, quals = zip(*[(path, qual) for (path, qual) in zip(path, quals) if qual > 0])
|
|
304
|
-
|
|
305
|
-
# convert units of path nodes from timesteps to seconds
|
|
306
|
-
path = [(audio_desc_timings[i], video_timings[j]) for (i,j) in path]
|
|
307
|
-
|
|
308
|
-
return path, quals
|
|
309
|
-
|
|
310
|
-
# chunk path segments of similar slope into clips
|
|
311
|
-
# a clip has the form: (start_index, end_index)
|
|
312
|
-
def chunk_path(smooth_path, tol):
|
|
313
|
-
x,y = zip(*smooth_path)
|
|
314
|
-
slopes = np.diff(y) / np.diff(x)
|
|
315
|
-
median_slope = np.median(slopes)
|
|
316
|
-
slope_changes = np.diff(slopes)
|
|
317
|
-
breaks = np.where(np.abs(slope_changes) > tol)[0] + 1
|
|
318
|
-
breaks = [0] + list(breaks) + [len(x)-1]
|
|
319
|
-
clips = list(zip(breaks[:-1], breaks[1:]))
|
|
320
|
-
return clips, median_slope, slopes
|
|
117
|
+
# ffmpeg command error handler
|
|
118
|
+
def run_ffmpeg_command(command, err_msg):
|
|
119
|
+
try:
|
|
120
|
+
return command.run(capture_stdout=True, capture_stderr=True, cmd=get_ffmpeg())
|
|
121
|
+
except ffmpeg.Error as e:
|
|
122
|
+
print(" ERROR: ffmpeg failed to " + err_msg)
|
|
123
|
+
print("FFmpeg error:")
|
|
124
|
+
print(e.stderr.decode('utf-8'))
|
|
125
|
+
raise
|
|
321
126
|
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
# stretch the x axis to make all slopes "cost" nearly the same
|
|
338
|
-
# without this, small changes to the slope at slope = +/-1
|
|
339
|
-
# cost sqrt(2) times as much as small changes at slope = 0
|
|
340
|
-
# by stretching, we limit the range of slopes to within +/- 1/x_stretch_factor
|
|
341
|
-
# the small angle approximation means these slopes all cost roughly the same
|
|
342
|
-
x_stretch_factor = 10.
|
|
343
|
-
rotated_stretched_path = [(x_stretch_factor*x,y) for x,y in rotated_path]
|
|
344
|
-
|
|
345
|
-
# L1-Minimization to solve the alignment problem using a linear program
|
|
346
|
-
# the absolute value functions needed for "absolute error" can be represented
|
|
347
|
-
# in a linear program by splitting variables into positive and negative pieces
|
|
348
|
-
# and constraining each to be positive (done by default in scipy's linprog)
|
|
349
|
-
# x is fit_err_pos, fit_err_neg, slope_change_pos, slope_change_neg
|
|
350
|
-
# fit_err[i] = path[i][1] - y_fit[i]
|
|
351
|
-
# slope_change[i] = (y_fit[i+2] - y_fit[i+1])/(path[i+2][0] - path[i+1][0]) - \
|
|
352
|
-
# (y_fit[i+1] - y_fit[i ])/(path[i+1][0] - path[i ][0])
|
|
353
|
-
# this can be rewritten in terms of fit_err by re-arranging the 1st equation:
|
|
354
|
-
# y_fit[i] = path[i][1] - fit_err[i]
|
|
355
|
-
# this gives:
|
|
356
|
-
# slope_change[i] = path_half[i] - fit_err_half[i]
|
|
357
|
-
# where each half is just the original equation but y_fit is swapped out
|
|
358
|
-
# the slope_change variables can then be set using equality constraints
|
|
359
|
-
num_fit_points = len(rotated_stretched_path)
|
|
360
|
-
x,y = [np.array(arr) for arr in zip(*rotated_stretched_path)]
|
|
361
|
-
x_diffs = np.diff(x, prepend=[-10**10], append=[10**10])
|
|
362
|
-
y_diffs = np.diff(y, prepend=[ 0 ], append=[ 0 ])
|
|
363
|
-
slope_change_magnitudes = np.abs(np.diff(y_diffs/x_diffs)) * x_stretch_factor
|
|
364
|
-
slope_change_locations = (slope_change_magnitudes > MAX_RATE_RATIO_DIFF_ALIGN)
|
|
365
|
-
slope_change_locations[1:-1] *= (np.abs(y[2:] - y[:-2]) > 5)
|
|
366
|
-
slope_change_costs = np.full(num_fit_points, smoothness / float(TIMESTEP_SIZE_SECONDS))
|
|
367
|
-
slope_change_costs[slope_change_locations] /= PREF_CUT_AT_GAPS_FACTOR
|
|
368
|
-
c = np.hstack([quals,
|
|
369
|
-
quals,
|
|
370
|
-
slope_change_costs * x_stretch_factor,
|
|
371
|
-
slope_change_costs * x_stretch_factor])
|
|
372
|
-
fit_err_coeffs = scipy.sparse.diags([ 1. / x_diffs[:-1],
|
|
373
|
-
-1. / x_diffs[:-1] - 1. / x_diffs[1:],
|
|
374
|
-
1. / x_diffs[1:]],
|
|
375
|
-
offsets=[0,1,2],
|
|
376
|
-
shape=(num_fit_points, num_fit_points + 2)).tocsc()[:,1:-1]
|
|
377
|
-
A_eq = scipy.sparse.hstack([ fit_err_coeffs,
|
|
378
|
-
-fit_err_coeffs,
|
|
379
|
-
scipy.sparse.eye(num_fit_points),
|
|
380
|
-
-scipy.sparse.eye(num_fit_points)])
|
|
381
|
-
b_eq = y_diffs[1: ] / x_diffs[1: ] - \
|
|
382
|
-
y_diffs[ :-1] / x_diffs[ :-1]
|
|
383
|
-
fit = scipy.optimize.linprog(c, A_eq=A_eq, b_eq=b_eq, method='highs-ds')
|
|
384
|
-
# if dual simplex solver encounters numerical problems, retry with interior point solver
|
|
385
|
-
if not fit.success and fit.status == 4:
|
|
386
|
-
fit = scipy.optimize.linprog(c, A_eq=A_eq, b_eq=b_eq, method='highs-ipm')
|
|
387
|
-
if not fit.success:
|
|
388
|
-
print(fit)
|
|
389
|
-
raise RuntimeError("Smooth Alignment L1-Min Optimization Failed!")
|
|
390
|
-
|
|
391
|
-
# combine fit_err_pos and fit_err_neg
|
|
392
|
-
fit_err = fit.x[:num_fit_points] - fit.x[num_fit_points:2*num_fit_points]
|
|
393
|
-
|
|
394
|
-
# subtract fit errors from nodes to retrieve the smooth fit's coordinates
|
|
395
|
-
# also, unstretch x axis and rotate basis back, reversing the affine pre-processing
|
|
396
|
-
smooth_path = [(((x / x_stretch_factor) - y) / 2.,
|
|
397
|
-
((x / x_stretch_factor) + y) / 2.) for x,y in zip(x, y - fit_err)]
|
|
398
|
-
|
|
399
|
-
# clip off start/end of replacement audio if it doesn't match or isn't aligned
|
|
400
|
-
# without this, describer intro/outro skips can cause mismatches at the start/end
|
|
401
|
-
# the problem would be localized and just means audio might not match video at the start/end
|
|
402
|
-
# instead we just keep the original video's audio in those segments if mismatches are detected
|
|
403
|
-
# if instead the first few or last few nodes are well-aligned, that edge is marked as synced
|
|
404
|
-
# during audio replacement, synced edges will be extended backwards/forwards as far as possible
|
|
405
|
-
# this is useful when the describer begins talking immediately (or before any alignable audio)
|
|
406
|
-
# or when the describer continues speaking until the end (or no more alignable audio remains)
|
|
407
|
-
# otherwise, the mismatch would result in the describer's voice not replacing audio in that part
|
|
408
|
-
max_sync_err = MAX_START_END_SYNC_ERR_SECONDS
|
|
409
|
-
smoothing_std = MIN_START_END_SYNC_TIME_SECONDS / (2. * TIMESTEP_SIZE_SECONDS)
|
|
410
|
-
smoothed_fit_err = nd.gaussian_filter(np.abs(fit_err), sigma=smoothing_std)
|
|
411
|
-
smooth_err_path = zip(smoothed_fit_err, smooth_path)
|
|
412
|
-
old_length = num_fit_points
|
|
413
|
-
smooth_err_path = list(itertools.dropwhile(lambda x: x[0] > max_sync_err, smooth_err_path))[::-1]
|
|
414
|
-
is_synced_at_start = len(smooth_err_path) == old_length
|
|
415
|
-
old_length = len(smooth_err_path)
|
|
416
|
-
smooth_err_path = list(itertools.dropwhile(lambda x: x[0] > max_sync_err, smooth_err_path))[::-1]
|
|
417
|
-
is_synced_at_end = len(smooth_err_path) == old_length
|
|
418
|
-
_, smooth_path = zip(*smooth_err_path)
|
|
419
|
-
smooth_path = list(smooth_path)
|
|
420
|
-
if is_synced_at_start:
|
|
421
|
-
slope = (smooth_path[1][1] - smooth_path[0][1]) / (smooth_path[1][0] - smooth_path[0][0])
|
|
422
|
-
smooth_path.insert(0, (-10e10, -10e10 * slope))
|
|
423
|
-
if is_synced_at_end:
|
|
424
|
-
slope = (smooth_path[-1][1] - smooth_path[-2][1]) / (smooth_path[-1][0] - smooth_path[-2][0])
|
|
425
|
-
smooth_path.append((10e10, 10e10 * slope))
|
|
426
|
-
|
|
427
|
-
clips, median_slope, slopes = chunk_path(smooth_path, tol=1e-7)
|
|
428
|
-
|
|
429
|
-
# assemble clips with slopes within the rate tolerance into runs
|
|
430
|
-
runs, run = [], []
|
|
431
|
-
bad_clips = []
|
|
432
|
-
for clip in clips:
|
|
433
|
-
if np.abs(median_slope-slopes[clip[0]]) > MAX_RATE_RATIO_DIFF_ALIGN:
|
|
434
|
-
if len(run) > 0:
|
|
435
|
-
runs.append(run)
|
|
436
|
-
run = []
|
|
437
|
-
bad_clips.append(clip)
|
|
438
|
-
continue
|
|
439
|
-
run.append(clip)
|
|
440
|
-
if len(run) > 0:
|
|
441
|
-
runs.append(run)
|
|
442
|
-
|
|
443
|
-
return smooth_path, runs, bad_clips, clips
|
|
127
|
+
def run_async_ffmpeg_command(command, media_arr, err_msg):
|
|
128
|
+
try:
|
|
129
|
+
ffmpeg_caller = command.run_async(pipe_stdin=True, quiet=True, cmd=get_ffmpeg())
|
|
130
|
+
out, err = ffmpeg_caller.communicate(media_arr.astype(np.int16).T.tobytes())
|
|
131
|
+
if len(err) > 0:
|
|
132
|
+
print(" ERROR: ffmpeg failed to " + err_msg)
|
|
133
|
+
print("FFmpeg error:")
|
|
134
|
+
print(err.decode('utf-8'))
|
|
135
|
+
raise ChildProcessError('FFmpeg error.')
|
|
136
|
+
except ffmpeg.Error as e:
|
|
137
|
+
print(" ERROR: ffmpeg failed to " + err_msg)
|
|
138
|
+
print("FFmpeg error:")
|
|
139
|
+
print(e.stderr.decode('utf-8'))
|
|
140
|
+
raise
|
|
444
141
|
|
|
445
|
-
#
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
audio_runtime = (audio_desc_arr.shape[1] - 2.) / AUDIO_SAMPLE_RATE
|
|
457
|
-
slope = smooth_path[-1][1] / smooth_path[-1][0]
|
|
458
|
-
new_end_point = (audio_runtime, smooth_path[-2][1] + (audio_runtime - smooth_path[-2][0]) * slope)
|
|
459
|
-
if new_end_point[1] > video_runtime:
|
|
460
|
-
new_end_point = (smooth_path[-2][0] + (video_runtime - smooth_path[-2][1]) / slope, video_runtime)
|
|
461
|
-
smooth_path[-1] = new_end_point
|
|
142
|
+
# read audio from file with ffmpeg and convert to numpy array
|
|
143
|
+
def parse_audio_from_file(media_file, num_channels=2):
|
|
144
|
+
# retrieve only the first audio track, injecting silence/trimming to force timestamps to match up
|
|
145
|
+
# for example, when the video starts before the audio this fills that starting gap with silence
|
|
146
|
+
ffmpeg_command = ffmpeg.input(media_file).output('-', format='s16le', acodec='pcm_s16le',
|
|
147
|
+
af='aresample=async=1:first_pts=0', map='0:a:0',
|
|
148
|
+
ac=num_channels, ar=AUDIO_SAMPLE_RATE, loglevel='error')
|
|
149
|
+
media_stream, _ = run_ffmpeg_command(ffmpeg_command, f"parse audio from input file: {media_file}")
|
|
150
|
+
# media_arr = np.frombuffer(media_stream, np.int16).astype(np.float32).reshape((-1, num_channels)).T
|
|
151
|
+
media_arr = np.frombuffer(media_stream, np.int16).astype(np.float16).reshape((-1, num_channels)).T
|
|
152
|
+
return media_arr
|
|
462
153
|
|
|
463
|
-
|
|
464
|
-
|
|
154
|
+
def plot_alignment(plot_filename_no_ext, path, audio_times, video_times, similarity_percent,
|
|
155
|
+
median_slope, stretch_audio, no_pitch_correction):
|
|
156
|
+
downsample = 20
|
|
157
|
+
path = path[::downsample]
|
|
158
|
+
video_times_full, audio_times_full, cluster_indices, quals, cum_quals = path.T
|
|
465
159
|
scatter_color = [.2,.4,.8]
|
|
466
160
|
lcs_rgba = np.zeros((len(quals),4))
|
|
467
161
|
lcs_rgba[:,:3] = np.array(scatter_color)[None,:]
|
|
468
|
-
lcs_rgba[:,3] = np.
|
|
469
|
-
|
|
162
|
+
lcs_rgba[:,3] = np.clip(quals * 400. / len(quals), 0, 1)
|
|
163
|
+
audio_offsets = audio_times_full - video_times_full
|
|
164
|
+
plt.switch_backend('Agg')
|
|
165
|
+
plt.scatter(video_times_full / 60., audio_offsets, s=3, c=lcs_rgba, label='Matches')
|
|
470
166
|
audio_offsets = audio_times - video_times
|
|
471
167
|
def expand_limits(start, end, ratio=.01):
|
|
472
168
|
average = (end + start) / 2.
|
|
@@ -474,62 +170,59 @@ def plot_alignment(plot_filename_no_ext, path, smooth_path, quals, runs, bad_cli
|
|
|
474
170
|
half_diff *= (1 + ratio)
|
|
475
171
|
return (average - half_diff, average + half_diff)
|
|
476
172
|
plt.xlim(expand_limits(*(0, np.max(video_times) / 60.)))
|
|
477
|
-
plt.ylim(expand_limits(*(np.min(audio_offsets) -
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
audio_times, video_times = np.array(smooth_path).T.reshape((2,-1))
|
|
481
|
-
audio_offsets = audio_times - video_times
|
|
482
|
-
if ad_timings is None:
|
|
173
|
+
plt.ylim(expand_limits(*(np.min(audio_offsets) - 10 * TIMESTEP_SIZE_SECONDS,
|
|
174
|
+
np.max(audio_offsets) + 10 * TIMESTEP_SIZE_SECONDS), .05))
|
|
175
|
+
if stretch_audio:
|
|
483
176
|
plt.plot(video_times / 60., audio_offsets, 'r-', lw=.5, label='Replaced Audio')
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
177
|
+
audio_times_unreplaced = []
|
|
178
|
+
video_times_unreplaced = []
|
|
179
|
+
for i in range(len(video_times) - 1):
|
|
180
|
+
slope = (audio_times[i+1] - audio_times[i]) / (video_times[i+1] - video_times[i])
|
|
181
|
+
if abs(1 - slope) > MAX_RATE_RATIO_DIFF_ALIGN:
|
|
182
|
+
video_times_unreplaced.extend(video_times[i:i+2])
|
|
183
|
+
audio_times_unreplaced.extend(audio_times[i:i+2])
|
|
184
|
+
video_times_unreplaced.append(video_times[i+1])
|
|
185
|
+
audio_times_unreplaced.append(np.nan)
|
|
186
|
+
if len(video_times_unreplaced) > 0:
|
|
187
|
+
video_times_unreplaced = np.array(video_times_unreplaced)
|
|
188
|
+
audio_times_unreplaced = np.array(audio_times_unreplaced)
|
|
189
|
+
audio_offsets = audio_times_unreplaced - video_times_unreplaced
|
|
190
|
+
plt.plot(video_times_unreplaced / 60., audio_offsets, 'c-', lw=1, label='Original Audio')
|
|
492
191
|
else:
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
video_times = ad_timings
|
|
498
|
-
audio_offsets = interp(ad_timings)
|
|
499
|
-
if len(audio_offsets) > 0:
|
|
500
|
-
plt.plot(video_times / 60., audio_offsets, 'r-', lw=1, label='Replaced Audio')
|
|
501
|
-
plt.xlabel('Video Time (minutes)')
|
|
502
|
-
plt.ylabel('Audio Description Offset (seconds)')
|
|
503
|
-
plt.title('Alignment')
|
|
192
|
+
plt.plot(video_times / 60., audio_offsets, 'r-', lw=1, label='Combined Media')
|
|
193
|
+
plt.xlabel('Original Video Time (minutes)')
|
|
194
|
+
plt.ylabel('Original Audio Description Offset (seconds behind video)')
|
|
195
|
+
plt.title(f"Alignment - Media Similarity {similarity_percent:.2f}%")
|
|
504
196
|
plt.legend().legend_handles[0].set_color(scatter_color)
|
|
505
197
|
plt.tight_layout()
|
|
506
198
|
plt.savefig(plot_filename_no_ext + '.png', dpi=400)
|
|
507
199
|
plt.clf()
|
|
508
|
-
|
|
509
200
|
with open(plot_filename_no_ext + '.txt', 'w') as file:
|
|
510
|
-
|
|
511
|
-
|
|
201
|
+
parameters = {'stretch_audio':stretch_audio, 'no_pitch_correction':no_pitch_correction}
|
|
202
|
+
print(f"Parameters: {parameters}", file=file)
|
|
203
|
+
this_script_path = os.path.abspath(__file__)
|
|
204
|
+
print(f"Version Hash: {get_version_hash(this_script_path)}", file=file)
|
|
205
|
+
video_offset = video_times[0] - audio_times[0]
|
|
206
|
+
print(f"Input file similarity: {similarity_percent:.2f}%", file=file)
|
|
512
207
|
print("Main changes needed to video to align it to audio input:", file=file)
|
|
513
208
|
print(f"Start Offset: {-video_offset:.2f} seconds", file=file)
|
|
514
209
|
print(f"Median Rate Change: {(median_slope-1.)*100:.2f}%", file=file)
|
|
515
|
-
for
|
|
516
|
-
|
|
517
|
-
audio_desc_end, video_end = smooth_path[clip_end]
|
|
518
|
-
slope = (video_end - video_start) / (audio_desc_end - audio_desc_start)
|
|
210
|
+
for i in range(len(video_times) - 1):
|
|
211
|
+
slope = (video_times[i+1] - video_times[i]) / (audio_times[i+1] - audio_times[i])
|
|
519
212
|
def str_from_time(seconds):
|
|
520
213
|
minutes, seconds = divmod(seconds, 60)
|
|
521
214
|
hours, minutes = divmod(minutes, 60)
|
|
522
|
-
return f"{hours:2.0f}:{minutes:02.0f}:{seconds:
|
|
523
|
-
print(f"Rate change of {(slope-1.)*100:
|
|
524
|
-
f"{str_from_time(
|
|
525
|
-
f"{str_from_time(
|
|
215
|
+
return f"{hours:2.0f}:{minutes:02.0f}:{seconds:06.3f}"
|
|
216
|
+
print(f"Rate change of {(slope-1.)*100:8.1f}% from {str_from_time(video_times[i])} to " + \
|
|
217
|
+
f"{str_from_time(video_times[i+1])} aligning with audio from " + \
|
|
218
|
+
f"{str_from_time(audio_times[i])} to {str_from_time(audio_times[i+1])}", file=file)
|
|
526
219
|
|
|
527
220
|
# use the smooth alignment to replace runs of video sound with corresponding described audio
|
|
528
|
-
def replace_aligned_segments(video_arr, audio_desc_arr,
|
|
221
|
+
def replace_aligned_segments(video_arr, audio_desc_arr, audio_desc_times, video_times, no_pitch_correction):
|
|
529
222
|
# perform quadratic interpolation of the audio description's waveform
|
|
530
223
|
# this allows it to be stretched to match the corresponding video segment
|
|
531
224
|
def audio_desc_arr_interp(samples):
|
|
532
|
-
chunk_size = 10**
|
|
225
|
+
chunk_size = 10**5
|
|
533
226
|
interpolated_chunks = []
|
|
534
227
|
for chunk in (samples[i:i+chunk_size] for i in range(0, len(samples), chunk_size)):
|
|
535
228
|
interp_bounds = (max(int(chunk[0]-2), 0),
|
|
@@ -538,210 +231,196 @@ def replace_aligned_segments(video_arr, audio_desc_arr, smooth_path, runs, no_pi
|
|
|
538
231
|
audio_desc_arr[:,slice(*interp_bounds)],
|
|
539
232
|
copy=False, bounds_error=False, fill_value=0,
|
|
540
233
|
kind='quadratic', assume_sorted=True)
|
|
541
|
-
interpolated_chunks.append(interp(chunk).astype(np.
|
|
234
|
+
interpolated_chunks.append(interp(chunk).astype(np.float16))
|
|
542
235
|
return np.hstack(interpolated_chunks)
|
|
543
236
|
|
|
544
|
-
#
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
237
|
+
# yields matrices of pearson correlations indexed by the first window's start and
|
|
238
|
+
# the second window's offset from the first window
|
|
239
|
+
# the output matrix is truncated to the valid square with positive offsets
|
|
240
|
+
# if negative=True, it is truncated to the valid square with negative offsets
|
|
241
|
+
# subsequent yields are the adjacent square following the previously yielded one
|
|
242
|
+
def get_pearson_corrs_generator(input, negative, jumps, window_size=512):
|
|
243
|
+
# processing the entire vector at once is faster, but uses too much memory
|
|
244
|
+
# instead, parse the input vector in pieces with a recursive call
|
|
245
|
+
max_cached_chunks = 50
|
|
246
|
+
cut = max_cached_chunks * window_size
|
|
247
|
+
if input.shape[1] > (max_cached_chunks + 2) * 1.1 * window_size:
|
|
248
|
+
is_first_iter = True
|
|
249
|
+
while True:
|
|
250
|
+
output_start = 0 if is_first_iter else 1
|
|
251
|
+
is_last_iter = (input.shape[1] <= (max_cached_chunks + 2) * 1.1 * window_size)
|
|
252
|
+
output_end = None if is_last_iter else max_cached_chunks
|
|
253
|
+
input_end = None if is_last_iter else (cut + window_size)
|
|
254
|
+
yield from itertools.islice(get_pearson_corrs_generator(input[:,:input_end], negative, jumps),
|
|
255
|
+
output_start, output_end)
|
|
256
|
+
if is_last_iter:
|
|
257
|
+
return
|
|
258
|
+
input = input[:,cut-window_size:]
|
|
259
|
+
is_first_iter = False
|
|
260
|
+
if input.shape[1] < 3 * window_size - 1:
|
|
261
|
+
raise RuntimeError("Invalid state in Pearson generator.")
|
|
262
|
+
pearson_corrs = np.zeros((len(jumps), input.shape[1] - window_size + 1)) - np.inf
|
|
263
|
+
# calculate dot products of pairs of windows (i.e. autocorrelation)
|
|
264
|
+
# avoids redundant calculations by substituting differences in the cumulative sum of products
|
|
265
|
+
self_corr = np.sum(input.astype(np.float32)**2, axis=0)
|
|
266
|
+
corr_cumsum = np.cumsum(self_corr, dtype=np.float64)
|
|
267
|
+
corr_cumsum[window_size:] -= corr_cumsum[:-window_size]
|
|
268
|
+
window_rms = corr_cumsum[window_size-1:]
|
|
269
|
+
epsilon = 1e-4 * max(1, np.max(window_rms))
|
|
270
|
+
window_rms = np.sqrt(window_rms + epsilon)
|
|
271
|
+
for jump_index, jump in enumerate(jumps):
|
|
272
|
+
autocorrelation = np.sum(input[:,jump:].astype(np.float32) * input[:,:input.shape[1]-jump], axis=0)
|
|
273
|
+
autocorr_cumsum = np.cumsum(autocorrelation, dtype=np.float64)
|
|
274
|
+
autocorr_cumsum[window_size:] -= autocorr_cumsum[:-window_size]
|
|
275
|
+
if negative:
|
|
276
|
+
pearson_corrs[jump_index, jump:] = autocorr_cumsum[window_size-1:] + epsilon
|
|
277
|
+
pearson_corrs[jump_index, jump:] /= window_rms[:len(window_rms)-jump]
|
|
570
278
|
else:
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
# identify which segments of the replaced audio actually have the describer speaking
|
|
580
|
-
# uses a Naive Bayes classifier smoothed with L1-Minimization to identify the describer
|
|
581
|
-
def detect_describer(video_arr, video_spec, video_spec_raw, video_timings,
|
|
582
|
-
smooth_path, detect_sensitivity, boost_sensitivity):
|
|
583
|
-
# retokenize the audio description, which has been stretched to match the video
|
|
584
|
-
audio_desc_spec_raw, audio_timings = tokenize_audio(video_arr)
|
|
585
|
-
audio_desc_spec = normalize_spec(audio_desc_spec_raw)
|
|
586
|
-
|
|
587
|
-
# avoid boosting or training on mismatched segments, like those close to skips
|
|
588
|
-
# assumes matching segments all have the same, constant play rate
|
|
589
|
-
# could be modified to handle a multi-modal distribution of rates
|
|
590
|
-
aligned_audio_times, aligned_video_times = zip(*smooth_path)
|
|
591
|
-
interp = scipy.interpolate.interp1d(aligned_video_times, aligned_audio_times,
|
|
592
|
-
fill_value = 'extrapolate',
|
|
593
|
-
bounds_error = False, assume_sorted = True)
|
|
594
|
-
slopes = (interp(video_timings + 1e-5) - \
|
|
595
|
-
interp(video_timings - 1e-5)) / 2e-5
|
|
596
|
-
median_slope = np.median(slopes)
|
|
597
|
-
aligned_mask = np.abs(slopes - median_slope) < MAX_RATE_RATIO_DIFF_ALIGN
|
|
598
|
-
well_aligned_mask = np.abs(slopes - median_slope) < MAX_RATE_RATIO_DIFF_BOOST
|
|
599
|
-
|
|
600
|
-
# first pass identification by assuming poorly matched tokens are describer speech
|
|
601
|
-
# also assumes the describer doesn't speak very quietly
|
|
602
|
-
corrs = np.sum(audio_desc_spec * video_spec, axis=-1)
|
|
603
|
-
smooth_volume = nd.gaussian_filter(audio_desc_spec[:,0], sigma=1)
|
|
604
|
-
audio_desc_loud = smooth_volume > np.percentile(smooth_volume, 30)
|
|
605
|
-
speech_mask = (corrs < .2) * audio_desc_loud
|
|
606
|
-
|
|
607
|
-
# normalize spectrogram coefficients along time axis to prep for conversion to PDFs
|
|
608
|
-
audio_desc_spec = normalize_spec(audio_desc_spec_raw, axes=(0,))
|
|
609
|
-
audio_desc_spec = np.clip(audio_desc_spec / 6., -1, 1)
|
|
610
|
-
video_spec = normalize_spec(video_spec_raw, axes=(0,))
|
|
611
|
-
video_spec = np.clip(video_spec / 6., -1, 1)
|
|
612
|
-
|
|
613
|
-
# convert sampled features (e.g. spectrogram) to probability densities of each feature
|
|
614
|
-
# when given a spectrogram, finds the distributions of the MFC coefficients
|
|
615
|
-
def make_log_pdfs(arr):
|
|
616
|
-
resolution = 100
|
|
617
|
-
bins_per_spot = 4
|
|
618
|
-
num_bins = int(resolution * bins_per_spot)
|
|
619
|
-
uniform_prior_strength_per_spot = 1
|
|
620
|
-
uniform_prior_strength_per_bin = uniform_prior_strength_per_spot / float(bins_per_spot)
|
|
621
|
-
bin_range = (-1 - 1e-10, 1 + 1e-10)
|
|
622
|
-
get_hist = lambda x: np.histogram(x, bins=num_bins, range=bin_range)[0]
|
|
623
|
-
pdfs = np.apply_along_axis(get_hist, 1, arr.T)
|
|
624
|
-
pdfs = pdfs + uniform_prior_strength_per_bin
|
|
625
|
-
smooth = lambda x: nd.gaussian_filter(x, sigma=bins_per_spot)
|
|
626
|
-
pdfs = np.apply_along_axis(smooth, 1, pdfs)
|
|
627
|
-
pdfs = pdfs / np.sum(pdfs[0,:])
|
|
628
|
-
log_pdfs = np.log(pdfs)
|
|
629
|
-
bin_edges = np.histogram([], bins=num_bins, range=bin_range)[1]
|
|
630
|
-
return log_pdfs, bin_edges
|
|
631
|
-
|
|
632
|
-
diff_spec = audio_desc_spec - video_spec
|
|
633
|
-
diff_spec = np.clip(diff_spec, -1, 1)
|
|
634
|
-
|
|
635
|
-
# Naive Bayes classifier to roughly estimate whether each token is describer speech
|
|
636
|
-
desc_log_pdfs, _ = make_log_pdfs(diff_spec[speech_mask * well_aligned_mask])
|
|
637
|
-
nondesc_log_pdfs, bin_edges = make_log_pdfs(diff_spec[(~speech_mask) * well_aligned_mask])
|
|
638
|
-
lratio_lookup = desc_log_pdfs - nondesc_log_pdfs
|
|
639
|
-
lratios = lratio_lookup[np.fromfunction(lambda i,j: j, diff_spec.shape, dtype=int),
|
|
640
|
-
np.digitize(diff_spec, bin_edges, right=True)-1]
|
|
641
|
-
ratio_desc_to_nondesc = np.sum(speech_mask * well_aligned_mask) /\
|
|
642
|
-
(np.sum((~speech_mask) * well_aligned_mask) + 1.)
|
|
643
|
-
relative_probs = np.sum(lratios, axis=1)
|
|
644
|
-
relative_probs /= np.std(relative_probs)
|
|
645
|
-
relative_probs -= np.mean(relative_probs)
|
|
279
|
+
pearson_corrs[jump_index, :pearson_corrs.shape[1]-jump] = autocorr_cumsum[window_size-1:] + epsilon
|
|
280
|
+
pearson_corrs[jump_index, :pearson_corrs.shape[1]-jump] /= window_rms[jump:]
|
|
281
|
+
# divide by RMS of constituent windows to get Pearson correlations
|
|
282
|
+
pearson_corrs = pearson_corrs / window_rms[None,:]
|
|
283
|
+
pearson_corrs = pearson_corrs.T
|
|
284
|
+
for chunk_index in range(0, input.shape[1] // window_size):
|
|
285
|
+
yield pearson_corrs[chunk_index*window_size:(chunk_index+1)*window_size]
|
|
646
286
|
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
287
|
+
def stretch(input, output, window_size=512, max_drift=512*3):
|
|
288
|
+
drift_window_size = max_drift * 2 + 1
|
|
289
|
+
num_input_samples = input.shape[1]
|
|
290
|
+
num_output_samples = output.shape[1]
|
|
291
|
+
total_offset_samples = num_output_samples - num_input_samples
|
|
292
|
+
jumps = [506, 451, 284, 410, 480, 379, 308, 430, 265, 494]
|
|
293
|
+
# use all jumps when given unreachable or difficult to reach offsets (i.e. Frobenius coin problem)
|
|
294
|
+
# otherwise, skip most jumps to trade off a little performance for a lot of speed
|
|
295
|
+
if abs(total_offset_samples) < 10000:
|
|
296
|
+
if abs(total_offset_samples) > 1000:
|
|
297
|
+
jumps.extend([MIN_STRETCH_OFFSET + offset for offset in (2**np.arange(8))-1])
|
|
298
|
+
else:
|
|
299
|
+
jumps = range(MIN_STRETCH_OFFSET, window_size)
|
|
300
|
+
num_windows = (num_input_samples // window_size)
|
|
301
|
+
window_to_offset = lambda window_index: (total_offset_samples * \
|
|
302
|
+
min((num_windows - 1), max(0, window_index))) // (num_windows - 1)
|
|
303
|
+
# note the absolute value in the drift
|
|
304
|
+
# the following calculations also use the absolute value of the jumps
|
|
305
|
+
# their signs flip together, so this saves on casework in the code
|
|
306
|
+
# after the optimal route is determined, the sign of the jumps will be reintroduced
|
|
307
|
+
window_to_offset_diff = lambda window_index: abs(window_to_offset(window_index) - \
|
|
308
|
+
window_to_offset(window_index - 1))
|
|
309
|
+
backpointers = np.zeros((num_windows, drift_window_size), dtype=np.int16)
|
|
310
|
+
best_jump_locations = np.zeros((num_windows, len(jumps)), dtype=np.int16)
|
|
311
|
+
cum_loss = np.zeros((3, drift_window_size)) + np.inf
|
|
312
|
+
cum_loss[1:, max_drift] = 0
|
|
313
|
+
last_offset_diff = 0
|
|
314
|
+
# if the output needs to be longer than the input, we need to jump backwards in the input
|
|
315
|
+
pearson_corrs_generator = get_pearson_corrs_generator(input, (total_offset_samples > 0), jumps)
|
|
316
|
+
for window_index in range(num_windows):
|
|
317
|
+
corrs = next(pearson_corrs_generator)
|
|
318
|
+
# for each jump distance, determine the best input index in the window to make that jump
|
|
319
|
+
best_jump_locations[window_index] = np.argmax(corrs, axis=0)
|
|
320
|
+
best_jump_losses = 1 - corrs[best_jump_locations[window_index], np.arange(corrs.shape[1])]
|
|
321
|
+
offset_diff = window_to_offset_diff(window_index)
|
|
322
|
+
offset_diff2 = offset_diff + last_offset_diff
|
|
323
|
+
offset_jump_losses = np.zeros((len(jumps)+1, drift_window_size)) + np.inf
|
|
324
|
+
# consider not jumping at all, copying the loss from the corresponding offset one window back
|
|
325
|
+
offset_jump_slice = slice(None, offset_jump_losses.shape[1] - offset_diff)
|
|
326
|
+
offset_jump_losses[0,offset_jump_slice] = cum_loss[(window_index-1)%3,offset_diff:]
|
|
327
|
+
for jump_index, jump in enumerate(jumps):
|
|
328
|
+
truncation_amount = offset_diff2 - jump
|
|
329
|
+
offset_jump_slice = slice(jump, drift_window_size - max(0, truncation_amount))
|
|
330
|
+
cum_loss_slice = slice(offset_diff2, drift_window_size + min(0, truncation_amount))
|
|
331
|
+
# consider jumping the given distance from two windows back
|
|
332
|
+
# a window is skipped when jumping to prevent overlapping crossfades
|
|
333
|
+
offset_jump_losses[jump_index+1, offset_jump_slice] = cum_loss[(window_index-2)%3, cum_loss_slice] + \
|
|
334
|
+
best_jump_losses[jump_index]
|
|
335
|
+
best_jumps = np.argmin(offset_jump_losses, axis=0)
|
|
336
|
+
backpointers[window_index] = best_jumps
|
|
337
|
+
cum_loss[window_index%3] = offset_jump_losses[best_jumps, np.arange(offset_jump_losses.shape[1])]
|
|
338
|
+
last_offset_diff = offset_diff
|
|
339
|
+
drift = max_drift
|
|
340
|
+
best_jumps = []
|
|
341
|
+
skip_window = False
|
|
342
|
+
for window_index in range(num_windows - 1, -1, -1):
|
|
343
|
+
drift += window_to_offset_diff(window_index + 1)
|
|
344
|
+
if skip_window:
|
|
345
|
+
skip_window = False
|
|
346
|
+
continue
|
|
347
|
+
best_jump_index = backpointers[window_index, drift] - 1
|
|
348
|
+
if best_jump_index == -1:
|
|
349
|
+
continue
|
|
350
|
+
best_jump = jumps[best_jump_index]
|
|
351
|
+
jump_input_index = window_index * window_size + \
|
|
352
|
+
best_jump_locations[window_index, best_jump_index].item()
|
|
353
|
+
drift -= best_jump
|
|
354
|
+
skip_window = True
|
|
355
|
+
best_jumps.append((jump_input_index, best_jump))
|
|
356
|
+
best_jumps = best_jumps[::-1]
|
|
357
|
+
best_jumps = np.array(best_jumps)
|
|
358
|
+
# reintroduce the sign of the jump distances
|
|
359
|
+
# if the output is longer, use backwards jumps in the input to duplicate samples
|
|
360
|
+
# if the output is shorter, use forwards jumps in the input to remove samples
|
|
361
|
+
if total_offset_samples > 0:
|
|
362
|
+
best_jumps[:,1] *= -1
|
|
363
|
+
jump_input_indices = best_jumps[:,0]
|
|
364
|
+
jump_distances = best_jumps[:,1]
|
|
365
|
+
# calculate starts and ends of segments that will be copied from input to output
|
|
366
|
+
input_starts = np.concatenate(([0], jump_input_indices + jump_distances))
|
|
367
|
+
input_ends = np.concatenate((jump_input_indices, [input.shape[1]]))
|
|
368
|
+
chunk_lengths = input_ends - input_starts
|
|
369
|
+
output_ends = np.cumsum(chunk_lengths)
|
|
370
|
+
output_starts = np.concatenate(([0], output_ends[:-1]))
|
|
371
|
+
bump = scipy.signal.windows.hann(2 * window_size + 1)
|
|
372
|
+
bump_head = bump[:window_size]
|
|
373
|
+
bump_tail = bump[window_size:-1]
|
|
374
|
+
output[:,:window_size] = input[:,:window_size]
|
|
375
|
+
for in_start, in_end, out_start, out_end in zip(input_starts, input_ends, output_starts, output_ends):
|
|
376
|
+
output[:,out_start:out_start+window_size] *= bump_tail
|
|
377
|
+
output[:,out_start:out_start+window_size] += input[:,in_start:in_start+window_size] * bump_head
|
|
378
|
+
output[:,out_start+window_size:out_end+window_size] = input[:,in_start+window_size:in_end+window_size]
|
|
715
379
|
|
|
716
|
-
|
|
380
|
+
x = audio_desc_times
|
|
381
|
+
y = video_times
|
|
382
|
+
x_samples = (x * AUDIO_SAMPLE_RATE).astype(int)
|
|
383
|
+
y_samples = (y * AUDIO_SAMPLE_RATE).astype(int)
|
|
384
|
+
diff_x_samples = np.diff(x_samples)
|
|
385
|
+
diff_y_samples = np.diff(y_samples)
|
|
386
|
+
slopes = diff_x_samples / diff_y_samples
|
|
387
|
+
total_offset_samples = diff_y_samples - diff_x_samples
|
|
388
|
+
y_midpoint_samples = (y_samples[:-1] + y_samples[1:]) // 2
|
|
389
|
+
progress_update_interval = (video_arr.shape[1] // 100) + 1
|
|
390
|
+
last_progress_update = -1
|
|
391
|
+
for i in range(len(x) - 1):
|
|
392
|
+
if diff_y_samples[i] < (MIN_DURATION_TO_REPLACE_SECONDS * AUDIO_SAMPLE_RATE) or \
|
|
393
|
+
np.abs(1 - slopes[i]) > MAX_RATE_RATIO_DIFF_ALIGN:
|
|
394
|
+
continue
|
|
395
|
+
video_arr_slice = video_arr[:,slice(*y_samples[i:i+2])]
|
|
396
|
+
progress = int(y_midpoint_samples[i] // progress_update_interval)
|
|
397
|
+
if progress > last_progress_update:
|
|
398
|
+
last_progress_update = progress
|
|
399
|
+
print(f" stretching audio:{progress:3d}% \r", end='')
|
|
400
|
+
# only apply pitch correction if the difference would be noticeable
|
|
401
|
+
if no_pitch_correction or np.abs(1 - slopes[i]) <= JUST_NOTICEABLE_DIFF_IN_FREQ_RATIO or \
|
|
402
|
+
abs(total_offset_samples[i]) < MIN_STRETCH_OFFSET:
|
|
403
|
+
# construct a stretched audio description waveform using the quadratic interpolator
|
|
404
|
+
sample_points = np.linspace(*x_samples[i:i+2], num=diff_y_samples[i], endpoint=False)
|
|
405
|
+
video_arr_slice[:] = audio_desc_arr_interp(sample_points)
|
|
406
|
+
else:
|
|
407
|
+
stretch(audio_desc_arr[:,slice(*x_samples[i:i+2])], video_arr_slice)
|
|
717
408
|
|
|
718
409
|
# Convert piece-wise linear fit to ffmpeg expression for editing video frame timestamps
|
|
719
|
-
def encode_fit_as_ffmpeg_expr(
|
|
410
|
+
def encode_fit_as_ffmpeg_expr(audio_desc_times, video_times, video_offset):
|
|
720
411
|
# PTS is the input frame's presentation timestamp, which is when frames are displayed
|
|
721
412
|
# TB is the timebase, which is how many seconds each unit of PTS corresponds to
|
|
722
413
|
# the output value of the expression will be the frame's new PTS
|
|
723
414
|
setts_cmd = ['TS']
|
|
724
|
-
start_skip = max(0, video_offset - start_key_frame)
|
|
725
|
-
if start_skip > 0:
|
|
726
|
-
# lossless cutting can only happen at key frames, so we cut the video before the audio starts
|
|
727
|
-
# but that means the video is behind the audio and needs to catch up by playing quicker
|
|
728
|
-
# catchup_spread is the ratio of time to spend catching up to the amount of catching up needed
|
|
729
|
-
catchup_spread = 1./CATCHUP_RATE
|
|
730
|
-
setts_cmd.append(f'+clip(TS-{start_key_frame},0,{start_skip*(1+catchup_spread)}/TB)*{-1./(1+catchup_spread)}')
|
|
731
|
-
elif video_offset < 0:
|
|
732
|
-
# if the audio starts before the video, stretch the first frame of the video back to meet it
|
|
733
|
-
setts_cmd.append(f'+clip(TS-{start_key_frame},0,{-video_offset/10000.}/TB)*10000')
|
|
734
415
|
# each segment of the linear fit can be encoded as a single clip function
|
|
735
416
|
setts_cmd.append('+(0')
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
slope = audio_desc_length / video_length
|
|
744
|
-
setts_cmd.append(f'+clip(TS-{start_key_frame}-{video_start:.4f}/TB,0,{max(0,video_length):.4f}/TB)*{slope-1:.9f}')
|
|
417
|
+
x = audio_desc_times
|
|
418
|
+
y = video_times
|
|
419
|
+
diff_x = np.diff(x)
|
|
420
|
+
diff_y = np.diff(y)
|
|
421
|
+
slopes = diff_x / diff_y
|
|
422
|
+
for i in range(len(audio_desc_times) - 1):
|
|
423
|
+
setts_cmd.append(f'+clip(TS-{y[i]-video_offset:.4f}/TB,0,{max(0,diff_y[i]):.4f}/TB)*{slopes[i]-1:.9f}')
|
|
745
424
|
setts_cmd.append(')')
|
|
746
425
|
setts_cmd = ''.join(setts_cmd)
|
|
747
426
|
return setts_cmd
|
|
@@ -752,75 +431,65 @@ def get_ffmpeg():
|
|
|
752
431
|
def get_ffprobe():
|
|
753
432
|
return static_ffmpeg.run._get_or_fetch_platform_executables_else_raise_no_lock()[1]
|
|
754
433
|
|
|
434
|
+
def get_key_frame_data(video_file, time=None, entry='pts_time'):
|
|
435
|
+
interval = f'%+{max(60,time+40)}' if time != None else '%'
|
|
436
|
+
key_frames = ffmpeg.probe(video_file, cmd=get_ffprobe(), select_streams='V', show_frames=None,
|
|
437
|
+
skip_frame='nokey', read_intervals=interval,
|
|
438
|
+
show_entries='frame='+entry)['frames']
|
|
439
|
+
return np.array([float(frame[entry]) for frame in key_frames if entry in frame])
|
|
440
|
+
|
|
441
|
+
# finds the average timestamp of (i.e. midpoint between) the key frames on either side of input time
|
|
755
442
|
def get_closest_key_frame_time(video_file, time):
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
443
|
+
key_frame_times = get_key_frame_data(video_file, time)
|
|
444
|
+
key_frame_times = key_frame_times if len(key_frame_times) > 0 else np.array([0])
|
|
445
|
+
prev_key_frame_times = key_frame_times[key_frame_times <= time]
|
|
446
|
+
prev_key_frame = np.max(prev_key_frame_times) if len(prev_key_frame_times) > 0 else time
|
|
447
|
+
next_key_frame_times = key_frame_times[key_frame_times > time]
|
|
448
|
+
next_key_frame = np.min(next_key_frame_times) if len(next_key_frame_times) > 0 else time
|
|
449
|
+
return (prev_key_frame + next_key_frame) / 2.
|
|
762
450
|
|
|
763
451
|
# outputs a new media file with the replaced audio (which includes audio descriptions)
|
|
764
452
|
def write_replaced_media_to_disk(output_filename, media_arr, video_file=None, audio_desc_file=None,
|
|
765
|
-
setts_cmd=None,
|
|
766
|
-
if
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
if
|
|
770
|
-
|
|
453
|
+
setts_cmd=None, video_offset=None, after_start_key_frame=None):
|
|
454
|
+
# if a media array is given, stretch_audio is enabled and media_arr should be added to the video
|
|
455
|
+
if media_arr is not None:
|
|
456
|
+
media_input = ffmpeg.input('pipe:', format='s16le', acodec='pcm_s16le', ac=2, ar=AUDIO_SAMPLE_RATE)
|
|
457
|
+
# if no video file is given, the input "video" was an audio file and the output should be too
|
|
458
|
+
if video_file is None:
|
|
459
|
+
write_command = ffmpeg.output(media_input, output_filename, loglevel='error').overwrite_output()
|
|
771
460
|
else:
|
|
772
|
-
original_video = ffmpeg.input(video_file)
|
|
461
|
+
original_video = ffmpeg.input(video_file, dn=None)
|
|
773
462
|
# "-max_interleave_delta 0" is sometimes necessary to fix an .mkv bug that freezes audio/video:
|
|
774
463
|
# ffmpeg bug warning: [matroska @ 0000000002c814c0] Starting new cluster due to timestamp
|
|
775
464
|
# more info about the bug and fix: https://reddit.com/r/ffmpeg/comments/efddfs/
|
|
776
465
|
write_command = ffmpeg.output(media_input, original_video, output_filename,
|
|
777
466
|
acodec='copy', vcodec='copy', scodec='copy',
|
|
778
|
-
max_interleave_delta='0', loglevel='
|
|
779
|
-
**{"c:a:0": "aac", "disposition:a:
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
467
|
+
max_interleave_delta='0', loglevel='error',
|
|
468
|
+
**{"c:a:0": "aac", "disposition:a:1": "original",
|
|
469
|
+
"metadata:s:a:1": "title=original",
|
|
470
|
+
"disposition:a:0": "default+visual_impaired+descriptions",
|
|
471
|
+
"metadata:s:a:0": "title=AD"}).overwrite_output()
|
|
472
|
+
run_async_ffmpeg_command(write_command, media_arr, f"write output file: {output_filename}")
|
|
784
473
|
else:
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
if os.path.splitext(
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
# mkv files break if 'nut' is used, while other files break when 'matroska' is used
|
|
805
|
-
format = 'matroska' if os.path.splitext(output_filename)[1] == '.mkv' else 'nut'
|
|
806
|
-
write_command = ffmpeg.output(original_video, 'pipe:', format=format, vsync='passthrough',
|
|
807
|
-
c='copy', loglevel='fatal')
|
|
808
|
-
ffmpeg_caller = write_command.run_async(pipe_stdout=True, cmd=get_ffmpeg())
|
|
809
|
-
pipe_input = ffmpeg.input('pipe:', format=format, thread_queue_size='512')
|
|
810
|
-
write_command2 = ffmpeg.output(media_input, pipe_input, output_filename, c='copy',
|
|
811
|
-
max_interleave_delta='0', loglevel='fatal', vsync='passthrough',
|
|
812
|
-
**{'bsf:v': f'setts=ts=\'{setts_cmd}\'',
|
|
813
|
-
'bsf:s': f'setts=ts=\'{setts_cmd}\''}).overwrite_output()
|
|
814
|
-
ffmpeg_caller2 = write_command2.run_async(pipe_stdin=True, cmd=get_ffmpeg())
|
|
815
|
-
while True:
|
|
816
|
-
in_bytes = ffmpeg_caller.stdout.read(100000)
|
|
817
|
-
if not in_bytes:
|
|
818
|
-
break
|
|
819
|
-
ffmpeg_caller2.stdin.write(in_bytes)
|
|
820
|
-
ffmpeg_caller2.stdin.close()
|
|
821
|
-
ffmpeg_caller.wait()
|
|
822
|
-
ffmpeg_caller2.wait()
|
|
823
|
-
|
|
474
|
+
start_offset = video_offset - after_start_key_frame
|
|
475
|
+
media_input = ffmpeg.input(audio_desc_file, itsoffset=f'{max(0, start_offset):.6f}')
|
|
476
|
+
original_video = ffmpeg.input(video_file, an=None, ss=f'{after_start_key_frame:.6f}',
|
|
477
|
+
itsoffset=f'{max(0, -start_offset):.6f}', dn=None)
|
|
478
|
+
# wav files don't have codecs compatible with most video containers, so we convert to aac
|
|
479
|
+
audio_codec = 'copy' if os.path.splitext(audio_desc_file)[1] != '.wav' else 'aac'
|
|
480
|
+
# flac audio may only have experimental support in some video containers (e.g. mp4)
|
|
481
|
+
standards = 'normal' if os.path.splitext(audio_desc_file)[1] != '.flac' else 'experimental'
|
|
482
|
+
# add frag_keyframe flag to prevent some players from ignoring audio/video start offsets
|
|
483
|
+
# set both pts and dts simultaneously in video manually, as ts= does not do the same thing
|
|
484
|
+
write_command = ffmpeg.output(media_input, original_video, output_filename,
|
|
485
|
+
acodec=audio_codec, vcodec='copy', scodec='copy',
|
|
486
|
+
max_interleave_delta='0', loglevel='error',
|
|
487
|
+
strict=standards, movflags='frag_keyframe',
|
|
488
|
+
**{'bsf:v': f'setts=pts=\'{setts_cmd}\':dts=\'{setts_cmd}\'',
|
|
489
|
+
'bsf:s': f'setts=ts=\'{setts_cmd}\'',
|
|
490
|
+
"disposition:a:0": "default+visual_impaired+descriptions",
|
|
491
|
+
"metadata:s:a:0": "title=AD"}).overwrite_output()
|
|
492
|
+
run_ffmpeg_command(write_command, f"write output file: {output_filename}")
|
|
824
493
|
|
|
825
494
|
# check whether static_ffmpeg has already installed ffmpeg and ffprobe
|
|
826
495
|
def is_ffmpeg_installed():
|
|
@@ -828,15 +497,482 @@ def is_ffmpeg_installed():
|
|
|
828
497
|
indicator_file = os.path.join(ffmpeg_dir, "installed.crumb")
|
|
829
498
|
return os.path.exists(indicator_file)
|
|
830
499
|
|
|
500
|
+
def get_energy(arr):
|
|
501
|
+
# downsample of 105, hann size 15, downsample by 2 gives 210 samples per second, ~65 halfwindows/second
|
|
502
|
+
decimation = 105
|
|
503
|
+
decimation2 = 2
|
|
504
|
+
arr_clip = arr[:,:(arr.shape[1] - (arr.shape[1] % decimation))].reshape(arr.shape[0], -1, decimation)
|
|
505
|
+
energy = np.einsum('ijk,ijk->j', arr_clip, arr_clip, dtype=np.float32) / (decimation * arr.shape[0])
|
|
506
|
+
hann_window = scipy.signal.windows.hann(15)[1:-1].astype(np.float32)
|
|
507
|
+
hann_window /= np.sum(hann_window)
|
|
508
|
+
energy_smooth = np.convolve(energy, hann_window, mode='same')
|
|
509
|
+
energy_smooth = np.log10(1 + energy_smooth) / 2.
|
|
510
|
+
return energy_smooth[::decimation2]
|
|
511
|
+
|
|
512
|
+
def get_zero_crossings(arr):
|
|
513
|
+
xings = np.diff(np.signbit(arr), prepend=False, axis=-1)
|
|
514
|
+
xings_clip = xings[:,:(xings.shape[1] - (xings.shape[1] % 210))].reshape(xings.shape[0], -1, 210)
|
|
515
|
+
zero_crossings = np.sum(np.abs(xings_clip), axis=(0,2)).astype(np.float32)
|
|
516
|
+
if xings.shape[0] == 1:
|
|
517
|
+
zero_crossings *= 2
|
|
518
|
+
hann_window = scipy.signal.windows.hann(15)[1:-1].astype(np.float32)
|
|
519
|
+
hann_window = hann_window / np.sum(hann_window)
|
|
520
|
+
zero_crossings_smooth = np.convolve(zero_crossings, hann_window, mode='same')
|
|
521
|
+
return zero_crossings_smooth
|
|
522
|
+
|
|
523
|
+
def downsample_blur(arr, downsample, blur):
|
|
524
|
+
hann_window = scipy.signal.windows.hann(downsample*blur+2)[1:-1].astype(np.float32)
|
|
525
|
+
hann_window = hann_window / np.sum(hann_window)
|
|
526
|
+
arr = arr[:len(arr)-(len(arr)%downsample)]
|
|
527
|
+
return sum((np.convolve(arr[i::downsample], hann_window[i::downsample],
|
|
528
|
+
mode='same') for i in range(downsample)))
|
|
529
|
+
|
|
530
|
+
def get_freq_bands(arr):
|
|
531
|
+
arr = np.mean(arr, axis=0) if arr.shape[0] > 1 else arr[0]
|
|
532
|
+
arr = arr[:len(arr)-(len(arr)%210)]
|
|
533
|
+
downsamples = [5, 7, 6]
|
|
534
|
+
decimation = 1
|
|
535
|
+
freq_bands = []
|
|
536
|
+
for downsample in downsamples:
|
|
537
|
+
if downsample == downsamples[-1]:
|
|
538
|
+
band_bottom = np.array(0).reshape(1)
|
|
539
|
+
else:
|
|
540
|
+
band_bottom = downsample_blur(arr, downsample, 3)
|
|
541
|
+
decimation *= downsample
|
|
542
|
+
arr = arr.reshape(-1, downsample)
|
|
543
|
+
band_energy = sum(((arr[:,i] - band_bottom) ** 2 for i in range(downsample)))
|
|
544
|
+
freq_band = downsample_blur(band_energy, (210 // decimation), 15) / 210
|
|
545
|
+
freq_band = np.log10(1 + freq_band) / 2.
|
|
546
|
+
freq_bands.append(freq_band)
|
|
547
|
+
arr = band_bottom
|
|
548
|
+
return freq_bands
|
|
549
|
+
|
|
550
|
+
def align(video_features, audio_desc_features, video_energy, audio_desc_energy):
|
|
551
|
+
samples_per_node = 210 // TIMESTEPS_PER_SECOND
|
|
552
|
+
hann_window_unnormed = scipy.signal.windows.hann(2*samples_per_node+1)[1:-1]
|
|
553
|
+
hann_window = hann_window_unnormed / np.sum(hann_window_unnormed)
|
|
554
|
+
get_mean = lambda arr: np.convolve(hann_window, arr, mode='same')
|
|
555
|
+
get_uniform_norm = lambda arr: np.convolve(np.ones(hann_window.shape), arr ** 2, mode='valid') ** .5
|
|
556
|
+
def get_uniform_norms(features):
|
|
557
|
+
return [np.clip(get_uniform_norm(feature), .001, None) for feature in features]
|
|
558
|
+
|
|
559
|
+
print(" memorizing video... \r", end='')
|
|
560
|
+
video_features_mean_sub = [feature - get_mean(feature) for feature in video_features]
|
|
561
|
+
audio_desc_features_mean_sub = [feature - get_mean(feature) for feature in audio_desc_features]
|
|
562
|
+
video_uniform_norms = get_uniform_norms(video_features_mean_sub)
|
|
563
|
+
audio_desc_uniform_norms = get_uniform_norms(audio_desc_features_mean_sub)
|
|
564
|
+
|
|
565
|
+
num_bins = 7
|
|
566
|
+
bin_spacing = 6
|
|
567
|
+
bins_width = (num_bins - 1) * bin_spacing + 1
|
|
568
|
+
bins_start = samples_per_node - 1 - (bins_width // 2)
|
|
569
|
+
bins_end = bins_start + bins_width
|
|
570
|
+
video_dicts = [defaultdict(set) for feature in video_features_mean_sub]
|
|
571
|
+
edges = np.array(np.meshgrid(*([np.arange(2)]*num_bins), indexing='ij')).reshape(num_bins,-1).T
|
|
572
|
+
bin_offsets = []
|
|
573
|
+
for edge in edges:
|
|
574
|
+
bin_offset = np.array(np.meshgrid(*[np.arange(x+1) for x in edge], indexing='ij'))
|
|
575
|
+
bin_offsets.append(np.dot(bin_offset.reshape(num_bins,-1)[::-1].T, 7**np.arange(num_bins)))
|
|
576
|
+
|
|
577
|
+
for video_dict, feature, norm in zip(video_dicts, video_features_mean_sub, video_uniform_norms):
|
|
578
|
+
bins = np.hstack([feature[bins_start+i:-bins_end+i+1, None] for i in bin_spacing * np.arange(num_bins)])
|
|
579
|
+
bins /= norm[:,None]
|
|
580
|
+
bins = 8 * bins + 3.3
|
|
581
|
+
np.clip(bins, 0, 6, out=bins)
|
|
582
|
+
bin_offset_indices = np.dot(((bins % 1) > .6), 2**np.arange(num_bins))
|
|
583
|
+
bins = np.dot(np.floor(bins).astype(int), 7**np.arange(num_bins)).tolist()
|
|
584
|
+
not_quiet = (video_energy[:-len(hann_window)] > .5)
|
|
585
|
+
for i in np.arange(len(video_energy) - len(hann_window))[not_quiet].tolist()[::4]:
|
|
586
|
+
bin = bins[i]
|
|
587
|
+
for bin_offset in bin_offsets[bin_offset_indices[i]].tolist():
|
|
588
|
+
video_dict[bin + bin_offset].add(i)
|
|
589
|
+
|
|
590
|
+
print(" matching audio... \r", end='')
|
|
591
|
+
audio_desc_bins = []
|
|
592
|
+
audio_desc_bin_offset_indices = []
|
|
593
|
+
for feature, norm in zip(audio_desc_features_mean_sub, audio_desc_uniform_norms):
|
|
594
|
+
bins = np.hstack([feature[bins_start+i:-bins_end+i+1, None] for i in bin_spacing * np.arange(num_bins)])
|
|
595
|
+
bins /= norm[:,None]
|
|
596
|
+
bins = 8 * bins + 3.5
|
|
597
|
+
bins = np.floor(bins).astype(int)
|
|
598
|
+
np.clip(bins, 0, 6, out=bins)
|
|
599
|
+
audio_desc_bins.append(np.dot(bins, 7**np.arange(num_bins)).tolist())
|
|
600
|
+
|
|
601
|
+
def pairwise_intersection(set1, set2, set3):
|
|
602
|
+
return (set1 & set2).union((set1 & set3), (set2 & set3))
|
|
603
|
+
def triwise_intersection(set1, set2, set3, set4, set5):
|
|
604
|
+
set123 = pairwise_intersection(set1, set2, set3)
|
|
605
|
+
return (set123 & set4) | (set123 & set5)
|
|
606
|
+
best_so_far = SortedList(key=lambda x:x[0])
|
|
607
|
+
best_so_far.add((-1,-1,0))
|
|
608
|
+
backpointers = {}
|
|
609
|
+
not_quiet = (audio_desc_energy[:-len(hann_window)] > .5)
|
|
610
|
+
for i in np.arange(len(audio_desc_energy) - len(hann_window))[not_quiet].tolist():
|
|
611
|
+
match_sets = [video_dict[bins[i]] for bins, video_dict in zip(audio_desc_bins, video_dicts)]
|
|
612
|
+
common = triwise_intersection(*match_sets)
|
|
613
|
+
match_points = []
|
|
614
|
+
for video_index in common:
|
|
615
|
+
prob = 1
|
|
616
|
+
for j in range(3):
|
|
617
|
+
corr = np.dot(audio_desc_features_mean_sub[j][i:i+2*samples_per_node-1],
|
|
618
|
+
video_features_mean_sub[j][video_index:video_index+2*samples_per_node-1])
|
|
619
|
+
corr /= audio_desc_uniform_norms[j][i] * video_uniform_norms[j][video_index]
|
|
620
|
+
prob *= max(1e-8, (1 - corr)) # Naive Bayes probability
|
|
621
|
+
prob = prob ** 2.9 # empirically determined, ranges from 2.5-3.4
|
|
622
|
+
if prob > 1e-8:
|
|
623
|
+
continue
|
|
624
|
+
qual = min(50, (prob / 1e-12) ** (-1. / 3)) # remove Naive Bayes assumption
|
|
625
|
+
match_points.append((video_index, qual))
|
|
626
|
+
audio_desc_index = i
|
|
627
|
+
for video_index, qual in sorted(match_points):
|
|
628
|
+
cur_index = best_so_far.bisect_right((video_index,))
|
|
629
|
+
prev_video_index, prev_audio_desc_index, prev_cum_qual = best_so_far[cur_index-1]
|
|
630
|
+
cum_qual = prev_cum_qual + qual
|
|
631
|
+
while (cur_index < len(best_so_far)) and (best_so_far[cur_index][2] <= cum_qual):
|
|
632
|
+
del best_so_far[cur_index]
|
|
633
|
+
best_so_far.add((video_index, audio_desc_index, cum_qual))
|
|
634
|
+
backpointers[(video_index, audio_desc_index)] = (prev_video_index, prev_audio_desc_index)
|
|
635
|
+
del video_dicts
|
|
636
|
+
path = [best_so_far[-1][:2]]
|
|
637
|
+
while path[-1][:2] in backpointers:
|
|
638
|
+
# failsafe to prevent an infinite loop that should never happen anyways
|
|
639
|
+
if len(path) > 10**8:
|
|
640
|
+
raise RuntimeError("Infinite Loop Encountered!")
|
|
641
|
+
path.append(backpointers[path[-1][:2]])
|
|
642
|
+
path.pop()
|
|
643
|
+
path.reverse()
|
|
644
|
+
if len(path) < max(min(len(video_energy), len(audio_desc_energy)) / 500., 5 * 210):
|
|
645
|
+
raise RuntimeError("Alignment failed, are the input files mismatched?")
|
|
646
|
+
y, x = np.array(path).T
|
|
647
|
+
|
|
648
|
+
half_hann_window = hann_window[:samples_per_node-1] / np.sum(hann_window[:samples_per_node-1])
|
|
649
|
+
half_samples_per_node = samples_per_node // 2
|
|
650
|
+
fit_delay = samples_per_node + half_samples_per_node - 2
|
|
651
|
+
diff_by = lambda arr, offset=half_samples_per_node: arr[offset:] - arr[:-offset]
|
|
652
|
+
def get_continuity_err(x, y, deriv=False):
|
|
653
|
+
x_smooth_future = np.convolve(x, half_hann_window, mode='valid')
|
|
654
|
+
y_smooth_future = np.convolve(y, half_hann_window, mode='valid')
|
|
655
|
+
slopes_future = diff_by(y_smooth_future) / diff_by(x_smooth_future)
|
|
656
|
+
offsets_future = y_smooth_future[:-half_samples_per_node] - \
|
|
657
|
+
x_smooth_future[:-half_samples_per_node] * slopes_future
|
|
658
|
+
x_smooth_past = np.convolve(x, half_hann_window[::-1], mode='valid')
|
|
659
|
+
y_smooth_past = np.convolve(y, half_hann_window[::-1], mode='valid')
|
|
660
|
+
slopes_past = diff_by(y_smooth_past) / diff_by(x_smooth_past)
|
|
661
|
+
offsets_past = y_smooth_past[half_samples_per_node:] - \
|
|
662
|
+
x_smooth_past[half_samples_per_node:] * slopes_past
|
|
663
|
+
continuity_err = np.full(len(x) - (1 if deriv else 0), np.inf)
|
|
664
|
+
fit_delay_offset = fit_delay - (1 if deriv else 0)
|
|
665
|
+
continuity_err[:-fit_delay_offset] = np.abs(slopes_future * x[:-fit_delay] + \
|
|
666
|
+
offsets_future - y[:-fit_delay])
|
|
667
|
+
continuity_err[fit_delay_offset:] = np.minimum(continuity_err[fit_delay_offset:],
|
|
668
|
+
np.abs(slopes_past * x[fit_delay:] + \
|
|
669
|
+
offsets_past - y[fit_delay:]))
|
|
670
|
+
return continuity_err
|
|
671
|
+
|
|
672
|
+
print(" refining match: pass 1 of 2...\r", end='')
|
|
673
|
+
continuity_err = get_continuity_err(x, y)
|
|
674
|
+
errs = (continuity_err < 3)
|
|
675
|
+
x = x[errs]
|
|
676
|
+
y = y[errs]
|
|
677
|
+
|
|
678
|
+
audio_desc_features_scaled = []
|
|
679
|
+
video_features_scaled = []
|
|
680
|
+
for video_feature, audio_desc_feature in zip(video_features, audio_desc_features):
|
|
681
|
+
audio_desc_feature_std = np.std(audio_desc_feature)
|
|
682
|
+
scale_factor = np.linalg.lstsq(video_feature[y][:,None], audio_desc_feature[x], rcond=None)[0]
|
|
683
|
+
audio_desc_features_scaled.append(audio_desc_feature / audio_desc_feature_std)
|
|
684
|
+
video_features_scaled.append(video_feature * scale_factor / audio_desc_feature_std)
|
|
685
|
+
audio_desc_features_scaled = np.array(list(zip(*(audio_desc_features_scaled[:3]))))
|
|
686
|
+
video_features_scaled = np.array(list(zip(*(video_features_scaled[:3]))))
|
|
687
|
+
|
|
688
|
+
smooth_x = get_mean(x)
|
|
689
|
+
smooth_y = get_mean(y)
|
|
690
|
+
slopes = np.diff(smooth_y) / np.diff(smooth_x)
|
|
691
|
+
offsets = smooth_y[:-1] - smooth_x[:-1] * slopes
|
|
692
|
+
err_y = slopes * x[:-1] + offsets - y[:-1]
|
|
693
|
+
compressed_x, compressed_y = [], []
|
|
694
|
+
def extend_all(index, compress=False, num=70):
|
|
695
|
+
compressed_x.extend([np.mean(x[index:index+num])] if compress else x[index:index+num])
|
|
696
|
+
compressed_y.extend([np.mean(y[index:index+num])] if compress else y[index:index+num])
|
|
697
|
+
extend_all(0, num=10)
|
|
698
|
+
for i in range(10, len(x) - 80, 70):
|
|
699
|
+
extend_all(i, compress=np.all(np.abs(err_y[i:i+70]) < 3))
|
|
700
|
+
extend_all(i+70)
|
|
701
|
+
|
|
702
|
+
x = compressed_x
|
|
703
|
+
y = compressed_y
|
|
704
|
+
|
|
705
|
+
match_dict = defaultdict(list)
|
|
706
|
+
x_unique = [-1]
|
|
707
|
+
for audio_desc_index, video_index in zip(x, y):
|
|
708
|
+
match_dict[audio_desc_index].append(video_index)
|
|
709
|
+
if audio_desc_index != x_unique[-1]:
|
|
710
|
+
x_unique.append(audio_desc_index)
|
|
711
|
+
x = np.array(x_unique[1:])
|
|
712
|
+
y = np.array([np.mean(match_dict[audio_desc_index]) for audio_desc_index in x])
|
|
713
|
+
|
|
714
|
+
# L1-Minimization to solve the alignment problem using a linear program
|
|
715
|
+
# the absolute value functions needed for "absolute error" can be represented
|
|
716
|
+
# in a linear program by splitting variables into positive and negative pieces
|
|
717
|
+
# and constraining each to be positive (done by default in scipy's linprog)
|
|
718
|
+
num_fit_points = len(x)
|
|
719
|
+
x_diffs = np.diff(x)
|
|
720
|
+
y_diffs = np.diff(y)
|
|
721
|
+
jump_cost_base = 10.
|
|
722
|
+
jump_costs = np.full(num_fit_points - 1, jump_cost_base)
|
|
723
|
+
continuity_err = get_continuity_err(x, y, deriv=True)
|
|
724
|
+
jump_costs /= np.maximum(1, np.sqrt(continuity_err / 3.))
|
|
725
|
+
rate_change_jump_costs = np.full(num_fit_points - 1, .001)
|
|
726
|
+
rate_change_costs = np.full(num_fit_points - 2, jump_cost_base * 4000)
|
|
727
|
+
shot_noise_costs = np.full(num_fit_points, .01)
|
|
728
|
+
shot_noise_jump_costs = np.full(num_fit_points - 1, 3)
|
|
729
|
+
shot_noise_bound = 2.
|
|
730
|
+
c = np.hstack([np.ones(2 * num_fit_points),
|
|
731
|
+
jump_costs,
|
|
732
|
+
jump_costs,
|
|
733
|
+
shot_noise_costs,
|
|
734
|
+
shot_noise_costs,
|
|
735
|
+
shot_noise_jump_costs,
|
|
736
|
+
shot_noise_jump_costs,
|
|
737
|
+
rate_change_jump_costs,
|
|
738
|
+
rate_change_jump_costs,
|
|
739
|
+
rate_change_costs,
|
|
740
|
+
rate_change_costs,
|
|
741
|
+
[0,]])
|
|
742
|
+
fit_err_coeffs = scipy.sparse.diags([-1. / x_diffs,
|
|
743
|
+
1. / x_diffs],
|
|
744
|
+
offsets=[0,1],
|
|
745
|
+
shape=(num_fit_points - 1, num_fit_points)).tocsc()
|
|
746
|
+
jump_coeffs = scipy.sparse.diags([ 1. / x_diffs],
|
|
747
|
+
offsets=[0],
|
|
748
|
+
shape=(num_fit_points - 1, num_fit_points - 1)).tocsc()
|
|
749
|
+
A_eq1 = scipy.sparse.hstack([ fit_err_coeffs,
|
|
750
|
+
-fit_err_coeffs,
|
|
751
|
+
jump_coeffs,
|
|
752
|
+
-jump_coeffs,
|
|
753
|
+
scipy.sparse.csc_matrix((num_fit_points - 1, 2 * num_fit_points)),
|
|
754
|
+
jump_coeffs,
|
|
755
|
+
-jump_coeffs,
|
|
756
|
+
jump_coeffs,
|
|
757
|
+
-jump_coeffs,
|
|
758
|
+
scipy.sparse.csc_matrix((num_fit_points - 1, 2 * num_fit_points - 4)),
|
|
759
|
+
np.ones((num_fit_points - 1, 1))])
|
|
760
|
+
A_eq2 = scipy.sparse.hstack([ scipy.sparse.csc_matrix((num_fit_points - 1, 4 * num_fit_points - 2)),
|
|
761
|
+
scipy.sparse.diags([-1., 1.], offsets=[0, 1],
|
|
762
|
+
shape=(num_fit_points - 1, num_fit_points)).tocsc(),
|
|
763
|
+
scipy.sparse.diags([1., -1.], offsets=[0, 1],
|
|
764
|
+
shape=(num_fit_points - 1, num_fit_points)).tocsc(),
|
|
765
|
+
-scipy.sparse.eye(num_fit_points - 1),
|
|
766
|
+
scipy.sparse.eye(num_fit_points - 1),
|
|
767
|
+
scipy.sparse.csc_matrix((num_fit_points - 1, 4 * num_fit_points - 6)),
|
|
768
|
+
scipy.sparse.csc_matrix((num_fit_points - 1, 1))])
|
|
769
|
+
slope_change_coeffs = scipy.sparse.diags([-1. / x_diffs[:-1],
|
|
770
|
+
1. / x_diffs[1:]],
|
|
771
|
+
offsets=[0,1],
|
|
772
|
+
shape=(num_fit_points - 2, num_fit_points - 1)).tocsc()
|
|
773
|
+
A_eq3 = scipy.sparse.hstack([scipy.sparse.csc_matrix((num_fit_points - 2, 8 * num_fit_points - 4)),
|
|
774
|
+
slope_change_coeffs,
|
|
775
|
+
-slope_change_coeffs,
|
|
776
|
+
-scipy.sparse.eye(num_fit_points - 2),
|
|
777
|
+
scipy.sparse.eye(num_fit_points - 2),
|
|
778
|
+
scipy.sparse.csc_matrix((num_fit_points - 2, 1))])
|
|
779
|
+
A_eq = scipy.sparse.vstack([A_eq1, A_eq2, A_eq3])
|
|
780
|
+
b_eq = y_diffs / x_diffs
|
|
781
|
+
b_eq = np.hstack((b_eq, np.zeros(2 * num_fit_points - 3)))
|
|
782
|
+
bounds = [[0, None]] * (4 * num_fit_points - 2) + \
|
|
783
|
+
[[0, shot_noise_bound]] * (2 * num_fit_points) + \
|
|
784
|
+
[[0, None]] * (6 * num_fit_points - 8) + \
|
|
785
|
+
[[None, None]]
|
|
786
|
+
fit = scipy.optimize.linprog(c, A_eq=A_eq, b_eq=b_eq, bounds=bounds, method='highs-ds')
|
|
787
|
+
# if dual simplex solver encounters numerical problems, retry with interior point solver
|
|
788
|
+
if not fit.success and fit.status == 4:
|
|
789
|
+
fit = scipy.optimize.linprog(c, A_eq=A_eq, b_eq=b_eq, bounds=bounds, method='highs-ipm')
|
|
790
|
+
if not fit.success:
|
|
791
|
+
print(fit)
|
|
792
|
+
raise RuntimeError("Smooth Alignment L1-Min Optimization Failed!")
|
|
793
|
+
|
|
794
|
+
# combine positive and negative components of variables
|
|
795
|
+
fit_err = fit.x[ : num_fit_points ] - \
|
|
796
|
+
fit.x[ num_fit_points :2*num_fit_points ]
|
|
797
|
+
slope_jumps = fit.x[8*num_fit_points-4: 9*num_fit_points-5] - \
|
|
798
|
+
fit.x[9*num_fit_points-5:10*num_fit_points-6]
|
|
799
|
+
median_slope = fit.x[-1]
|
|
800
|
+
slopes = median_slope + (slope_jumps / x_diffs)
|
|
801
|
+
|
|
802
|
+
# subtract fit errors from nodes to retrieve the smooth fit's coordinates
|
|
803
|
+
smooth_path = [(x, y) for x,y in zip(x, y - fit_err)]
|
|
804
|
+
|
|
805
|
+
print(" refining match: pass 2 of 2...\r", end='')
|
|
806
|
+
slopes_plus_ends = np.hstack((slopes[:1], slopes, slopes[-1:]))
|
|
807
|
+
extensions = []
|
|
808
|
+
extend_radius = 210 * 30 # +/- 30 seconds
|
|
809
|
+
video_interp = scipy.interpolate.make_interp_spline(np.arange(len(video_features_scaled)),
|
|
810
|
+
video_features_scaled, k=1)
|
|
811
|
+
colinear_dict = defaultdict(list)
|
|
812
|
+
for i, (x, y) in enumerate(smooth_path):
|
|
813
|
+
for slope in slopes_plus_ends[i:i+2]:
|
|
814
|
+
if (slope < .1) or (slope > 10):
|
|
815
|
+
continue
|
|
816
|
+
offset = y - slope * x
|
|
817
|
+
colinear_dict[(round(slope, 6), int(round(offset, 0)))].append((x, y))
|
|
818
|
+
line_clusters = []
|
|
819
|
+
added_keys = set()
|
|
820
|
+
for (slope, offset), indices in sorted(colinear_dict.items(), key=lambda x: -len(x[1])):
|
|
821
|
+
if (slope, offset) in added_keys:
|
|
822
|
+
continue
|
|
823
|
+
line_clusters.append(indices)
|
|
824
|
+
added_keys.add((slope, offset))
|
|
825
|
+
del colinear_dict[(slope, offset)]
|
|
826
|
+
for (slope2, offset2), indices2 in list(colinear_dict.items()):
|
|
827
|
+
if (abs(indices2[ 0][1] - (indices2[ 0][0] * slope + offset)) < 3) and \
|
|
828
|
+
(abs(indices2[-1][1] - (indices2[-1][0] * slope + offset)) < 3):
|
|
829
|
+
line_clusters[-1].extend(colinear_dict[(slope2, offset2)])
|
|
830
|
+
added_keys.add((slope2, offset2))
|
|
831
|
+
del colinear_dict[(slope2, offset2)]
|
|
832
|
+
line_clusters = [sorted(cluster) for cluster in line_clusters]
|
|
833
|
+
line_clusters = [x for x in line_clusters if (abs(x[0][0] - x[-1][0]) > 10) and len(x) > 5]
|
|
834
|
+
|
|
835
|
+
for i, cluster in enumerate(line_clusters):
|
|
836
|
+
x, y = np.array(cluster).T
|
|
837
|
+
linear_fit = np.linalg.lstsq(np.hstack((np.ones((len(x), 1)), x[:, None])), y, rcond=None)[0]
|
|
838
|
+
line_clusters[i] = (x, linear_fit[0], linear_fit[1])
|
|
839
|
+
|
|
840
|
+
def get_x_limits(x, offset, slope, extend_horiz=extend_radius, buffer_vert=4):
|
|
841
|
+
limits = (max(int(x[0]) - extend_horiz, 0),
|
|
842
|
+
min(int(x[-1]) + extend_horiz, len(audio_desc_features_scaled) - 1))
|
|
843
|
+
limits = (max(limits[0], int(np.ceil((buffer_vert - offset) / slope))),
|
|
844
|
+
min(limits[1], int(np.floor((len(video_features_scaled) - buffer_vert - offset) / slope))))
|
|
845
|
+
return limits
|
|
846
|
+
def get_audio_video_matches(limits, slope, offset):
|
|
847
|
+
x = np.arange(*limits)
|
|
848
|
+
y = slope * x + offset
|
|
849
|
+
audio_match = audio_desc_features_scaled[slice(*limits)]
|
|
850
|
+
video_match = video_interp(y)
|
|
851
|
+
return x, y, audio_match, video_match
|
|
852
|
+
|
|
853
|
+
audio_desc_max_energy = np.max(audio_desc_features_scaled[:,0])
|
|
854
|
+
video_max_energy = np.max(video_features_scaled[:,0])
|
|
855
|
+
points = [[] for i in range(len(audio_desc_features_scaled))]
|
|
856
|
+
seen_points = set()
|
|
857
|
+
for cluster_index, (x, offset, slope) in enumerate(line_clusters):
|
|
858
|
+
limits = get_x_limits(x, offset, slope, extend_horiz=0)
|
|
859
|
+
if limits[1] < limits[0] + 5:
|
|
860
|
+
continue
|
|
861
|
+
if limits[1] > limits[0] + 100:
|
|
862
|
+
x, y, audio_match, video_match = get_audio_video_matches(limits, slope, offset)
|
|
863
|
+
video_match_err = audio_match[1:-1] - video_match[1:-1]
|
|
864
|
+
valid_matches = np.mean(video_match_err, axis=-1) < 0.1
|
|
865
|
+
if np.count_nonzero(valid_matches) > 50:
|
|
866
|
+
video_match_diff = (video_match[2:] - video_match[:-2]) / 2.
|
|
867
|
+
video_match_err = video_match_err[valid_matches]
|
|
868
|
+
video_match_diff = video_match_diff[valid_matches]
|
|
869
|
+
x_valid = x[1:-1][valid_matches][:,None]
|
|
870
|
+
A = video_match_diff.reshape(-1,1)
|
|
871
|
+
linear_fit, residual, _, _ = np.linalg.lstsq(A, video_match_err.flat, rcond=None)
|
|
872
|
+
explained_err_ratio = 1 - (residual / np.sum(video_match_err ** 2))
|
|
873
|
+
stds_above_noise_mean = np.sqrt(explained_err_ratio * np.prod(video_match_err.shape)) - 1.
|
|
874
|
+
if stds_above_noise_mean > 8 and abs(linear_fit[0]) < 2:
|
|
875
|
+
offset += linear_fit[0]
|
|
876
|
+
limits = get_x_limits(x, offset, slope)
|
|
877
|
+
x, y, audio_match, video_match = get_audio_video_matches(limits, slope, offset)
|
|
878
|
+
quals = np.sum(-.5 - np.log10(1e-4 + np.abs(audio_match - video_match)), axis=1)
|
|
879
|
+
quals *= np.clip(video_match[:,0] + 2.5 - video_max_energy, 0, 1)
|
|
880
|
+
quals += np.clip(audio_match[:,0] + 2.5 - audio_desc_max_energy, 0, 1) * .1
|
|
881
|
+
energy_diffs = audio_match[:,0] - video_match[:,0]
|
|
882
|
+
for i, j, qual in zip(x.tolist(), y.tolist(), quals.tolist()):
|
|
883
|
+
point = (i, int(j))
|
|
884
|
+
if point not in seen_points:
|
|
885
|
+
seen_points.add(point)
|
|
886
|
+
points[i].append((j, cluster_index, qual))
|
|
887
|
+
del seen_points
|
|
888
|
+
points = [sorted(point) for point in points]
|
|
889
|
+
|
|
890
|
+
best_so_far = SortedList(key=lambda x:x[0])
|
|
891
|
+
best_so_far.add((0, 0, -1, 0, 0)) # video_index, audio_desc_index, cluster_index, qual, cum_qual
|
|
892
|
+
clusters_best_so_far = [(0, 0, 0, -1000) for cluster in line_clusters]
|
|
893
|
+
backpointers = {}
|
|
894
|
+
prev_cache = np.full((len(video_features_scaled), 5), -np.inf)
|
|
895
|
+
prev_cache[0] = (0, 0, -1, 0, 0) # video_index, audio_desc_index, cluster_index, qual, cum_qual
|
|
896
|
+
reversed_min_points = [min(x)[0] if len(x) > 0 else np.inf for x in points[::-1]]
|
|
897
|
+
forward_min = list(itertools.accumulate(reversed_min_points, min))[::-1]
|
|
898
|
+
for i in range(len(audio_desc_features_scaled)):
|
|
899
|
+
for j, cluster_index, qual in points[i]:
|
|
900
|
+
cur_index = best_so_far.bisect_right((j,))
|
|
901
|
+
prev_j, prev_i, prev_cluster_index, prev_qual, best_prev_cum_qual = best_so_far[cur_index-1]
|
|
902
|
+
cluster_last = clusters_best_so_far[cluster_index]
|
|
903
|
+
if cluster_last[3] >= best_prev_cum_qual:
|
|
904
|
+
prev_j, prev_i, prev_qual, best_prev_cum_qual = cluster_last
|
|
905
|
+
prev_cluster_index = cluster_index
|
|
906
|
+
for prev_j_temp in range(max(0, int(j) - 2), int(j) + 1):
|
|
907
|
+
prev_node = prev_cache[prev_j_temp].tolist()
|
|
908
|
+
if cluster_index != prev_node[2]:
|
|
909
|
+
prev_node[4] -= 100 + 100 * ((j - prev_node[0]) - (i - prev_node[1])) ** 2
|
|
910
|
+
if prev_node[1] >= (i - 2) and \
|
|
911
|
+
prev_node[0] <= j and \
|
|
912
|
+
prev_node[4] >= best_prev_cum_qual:
|
|
913
|
+
prev_j, prev_i, prev_cluster_index, prev_qual, best_prev_cum_qual = prev_node
|
|
914
|
+
cum_qual = best_prev_cum_qual + qual
|
|
915
|
+
prev_cache[int(j)] = (j, i, cluster_index, qual, cum_qual)
|
|
916
|
+
cum_qual_jump = cum_qual - 1000
|
|
917
|
+
if best_so_far[cur_index-1][4] < cum_qual_jump:
|
|
918
|
+
while (cur_index < len(best_so_far)) and (best_so_far[cur_index][4] <= cum_qual_jump):
|
|
919
|
+
del best_so_far[cur_index]
|
|
920
|
+
best_so_far.add((j, i, cluster_index, qual, cum_qual_jump))
|
|
921
|
+
if forward_min[i] == j and cur_index > 1:
|
|
922
|
+
del best_so_far[:cur_index-1]
|
|
923
|
+
cum_qual_cluster_jump = cum_qual - 50
|
|
924
|
+
if cluster_last[3] < cum_qual_cluster_jump:
|
|
925
|
+
clusters_best_so_far[cluster_index] = (j, i, qual, cum_qual_cluster_jump)
|
|
926
|
+
backpointers[(j, i)] = (prev_j, prev_i, prev_cluster_index, prev_qual, best_prev_cum_qual)
|
|
927
|
+
path = [best_so_far[-1]]
|
|
928
|
+
while path[-1][:2] in backpointers:
|
|
929
|
+
path.append(backpointers[path[-1][:2]])
|
|
930
|
+
path.pop()
|
|
931
|
+
path.reverse()
|
|
932
|
+
path = np.array(path)
|
|
933
|
+
y, x, cluster_indices, quals, cum_quals = path.T
|
|
934
|
+
|
|
935
|
+
nondescription = ((quals == 0) | (quals > .3))
|
|
936
|
+
similarity_ratio_x = float(len(set(x[nondescription]))) / len(audio_desc_features_scaled)
|
|
937
|
+
similarity_ratio_y = float(len(set(y[nondescription]))) / len(video_features_scaled)
|
|
938
|
+
similarity_percent = 100 * max(similarity_ratio_x, similarity_ratio_y)
|
|
939
|
+
|
|
940
|
+
nodes = []
|
|
941
|
+
if cluster_indices[0] == cluster_indices[1]:
|
|
942
|
+
nodes.append((x[0], y[0]))
|
|
943
|
+
for i in range(len(x) - 1):
|
|
944
|
+
if cluster_indices[i] != cluster_indices[i+1]:
|
|
945
|
+
nodes.append((x[i] - .1, y[i] - .1))
|
|
946
|
+
nodes.append((x[i+1] + .1, y[i+1] + .1))
|
|
947
|
+
if cluster_indices[-2] == cluster_indices[-1]:
|
|
948
|
+
nodes.append((x[-1], y[-1]))
|
|
949
|
+
x, y = np.array(nodes).T / 210.
|
|
950
|
+
|
|
951
|
+
if (x[1] - x[0]) > 2:
|
|
952
|
+
slope_start = (y[1] - y[0]) / (x[1] - x[0])
|
|
953
|
+
x[0] = 0
|
|
954
|
+
y[0] = y[1] - (x[1] * slope_start)
|
|
955
|
+
if y[0] < 0:
|
|
956
|
+
x[0] = x[1] - (y[1] / slope_start)
|
|
957
|
+
y[0] = 0
|
|
958
|
+
if (x[-1] - x[-2]) > 2:
|
|
959
|
+
slope_end = (y[-1] - y[-2]) / (x[-1] - x[-2])
|
|
960
|
+
x[-1] = ((len(audio_desc_energy) - 1) / 210.)
|
|
961
|
+
y[-1] = y[-2] + ((x[-1] - x[-2]) * slope_end)
|
|
962
|
+
if y[-1] > ((len(video_energy) - 1) / 210.):
|
|
963
|
+
y[-1] = ((len(video_energy) - 1) / 210.)
|
|
964
|
+
x[-1] = x[-2] + ((y[-1] - y[-2]) / slope_end)
|
|
965
|
+
|
|
966
|
+
path[:,:2] /= 210.
|
|
967
|
+
return x, y, similarity_percent, path, median_slope
|
|
968
|
+
|
|
831
969
|
# combines videos with matching audio files (e.g. audio descriptions)
|
|
832
970
|
# this is the main function of this script, it calls the other functions in order
|
|
833
|
-
def combine(video, audio,
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
alignment_dir=default_alignment_dir, extension="copy", display_func=None):
|
|
837
|
-
video_files, video_file_types = get_sorted_filenames(video, VIDEO_EXTENSIONS, AUDIO_EXTENSIONS)
|
|
971
|
+
def combine(video, audio, stretch_audio=False, yes=False, prepend="ad_", no_pitch_correction=False,
|
|
972
|
+
output_dir=default_output_dir, alignment_dir=default_alignment_dir):
|
|
973
|
+
video_files, has_audio_extensions = get_sorted_filenames(video, VIDEO_EXTENSIONS, AUDIO_EXTENSIONS)
|
|
838
974
|
|
|
839
|
-
if yes == False and sum(
|
|
975
|
+
if yes == False and sum(has_audio_extensions) > 0:
|
|
840
976
|
print("")
|
|
841
977
|
print("One or more audio files found in video input. Was this intentional?")
|
|
842
978
|
print("If not, press ctrl+c to kill this script.")
|
|
@@ -849,16 +985,16 @@ def combine(video, audio, smoothness=50, stretch_audio=False, keep_non_ad=False,
|
|
|
849
985
|
f"The audio path has {len(audio_desc_files)} files"]
|
|
850
986
|
raise RuntimeError("\n".join(error_msg))
|
|
851
987
|
|
|
852
|
-
|
|
853
|
-
ensure_folders_exist([output_dir]
|
|
988
|
+
print("")
|
|
989
|
+
ensure_folders_exist([output_dir])
|
|
854
990
|
if PLOT_ALIGNMENT_TO_FILE:
|
|
855
|
-
ensure_folders_exist([alignment_dir]
|
|
991
|
+
ensure_folders_exist([alignment_dir])
|
|
856
992
|
|
|
857
|
-
|
|
993
|
+
print("")
|
|
858
994
|
for (video_file, audio_desc_file) in zip(video_files, audio_desc_files):
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
995
|
+
print(os.path.split(video_file)[1])
|
|
996
|
+
print(os.path.split(audio_desc_file)[1])
|
|
997
|
+
print("")
|
|
862
998
|
if yes == False:
|
|
863
999
|
print("Are the above input file pairings correct?")
|
|
864
1000
|
print("If not, press ctrl+c to kill this script.")
|
|
@@ -867,396 +1003,710 @@ def combine(video, audio, smoothness=50, stretch_audio=False, keep_non_ad=False,
|
|
|
867
1003
|
|
|
868
1004
|
# if ffmpeg isn't installed, install it
|
|
869
1005
|
if not is_ffmpeg_installed():
|
|
870
|
-
|
|
1006
|
+
print("Downloading and installing ffmpeg (media editor, 50 MB download)...")
|
|
871
1007
|
get_ffmpeg()
|
|
872
1008
|
if not is_ffmpeg_installed():
|
|
873
1009
|
RuntimeError("Failed to install ffmpeg.")
|
|
874
|
-
|
|
1010
|
+
print("Successfully installed ffmpeg.")
|
|
875
1011
|
|
|
876
|
-
|
|
1012
|
+
print("Processing files:")
|
|
877
1013
|
|
|
878
|
-
for (video_file, audio_desc_file,
|
|
879
|
-
|
|
880
|
-
#
|
|
881
|
-
|
|
882
|
-
ext = os.path.splitext(video_file)[1]
|
|
883
|
-
else:
|
|
884
|
-
# add a dot to the extension if it's missing
|
|
885
|
-
ext = ('' if extension[0] == '.' else '.') + extension
|
|
886
|
-
output_filename = prepend + os.path.splitext(os.path.split(video_file)[1])[0] + ext
|
|
1014
|
+
for (video_file, audio_desc_file, has_audio_extension) in zip(video_files, audio_desc_files,
|
|
1015
|
+
has_audio_extensions):
|
|
1016
|
+
# Output filename (and extension) is the same as input, except the prepend and directory
|
|
1017
|
+
output_filename = prepend + os.path.split(video_file)[1]
|
|
887
1018
|
output_filename = os.path.join(output_dir, output_filename)
|
|
888
|
-
|
|
1019
|
+
print(f" {output_filename}")
|
|
889
1020
|
|
|
890
|
-
if
|
|
891
|
-
|
|
1021
|
+
if (not stretch_audio) & has_audio_extension:
|
|
1022
|
+
raise RuntimeError("Argument --stretch_audio is required when both inputs are audio files.")
|
|
1023
|
+
|
|
1024
|
+
if os.path.exists(output_filename) and os.path.getsize(output_filename) > 1e5:
|
|
1025
|
+
print(" output file already exists, skipping...")
|
|
892
1026
|
continue
|
|
893
1027
|
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
1028
|
+
# print warning if output file's full path is longer than Windows MAX_PATH (260)
|
|
1029
|
+
full_output_filename = os.path.abspath(output_filename)
|
|
1030
|
+
if IS_RUNNING_WINDOWS and len(full_output_filename) >= 260:
|
|
1031
|
+
print(" WARNING: very long output path, ffmpeg may fail...")
|
|
1032
|
+
|
|
1033
|
+
num_channels = 2 if stretch_audio else 1
|
|
1034
|
+
print(" reading video file...\r", end='')
|
|
1035
|
+
video_arr = parse_audio_from_file(video_file, num_channels)
|
|
900
1036
|
|
|
901
|
-
|
|
902
|
-
|
|
1037
|
+
print(" computing video features... \r", end='')
|
|
1038
|
+
video_energy = get_energy(video_arr)
|
|
1039
|
+
video_zero_crossings = get_zero_crossings(video_arr)
|
|
1040
|
+
video_freq_bands = get_freq_bands(video_arr)
|
|
1041
|
+
video_features = [video_energy, video_zero_crossings] + video_freq_bands
|
|
903
1042
|
|
|
904
|
-
|
|
1043
|
+
if not stretch_audio:
|
|
1044
|
+
del video_arr
|
|
905
1045
|
|
|
906
|
-
|
|
1046
|
+
print(" reading audio file... \r", end='')
|
|
1047
|
+
audio_desc_arr = parse_audio_from_file(audio_desc_file, num_channels)
|
|
1048
|
+
|
|
1049
|
+
print(" computing audio features...\r", end='')
|
|
1050
|
+
audio_desc_energy = get_energy(audio_desc_arr)
|
|
1051
|
+
audio_desc_zero_crossings = get_zero_crossings(audio_desc_arr)
|
|
1052
|
+
audio_desc_freq_bands = get_freq_bands(audio_desc_arr)
|
|
1053
|
+
audio_desc_features = [audio_desc_energy, audio_desc_zero_crossings] + audio_desc_freq_bands
|
|
1054
|
+
|
|
1055
|
+
if not stretch_audio:
|
|
1056
|
+
del audio_desc_arr
|
|
907
1057
|
|
|
908
|
-
|
|
1058
|
+
outputs = align(video_features, audio_desc_features, video_energy, audio_desc_energy)
|
|
1059
|
+
audio_desc_times, video_times, similarity_percent, path, median_slope = outputs
|
|
1060
|
+
|
|
1061
|
+
del video_energy, video_zero_crossings, video_freq_bands, video_features
|
|
1062
|
+
del audio_desc_energy, audio_desc_zero_crossings, audio_desc_freq_bands, audio_desc_features
|
|
1063
|
+
|
|
1064
|
+
if similarity_percent < 20:
|
|
1065
|
+
print(f" WARNING: similarity {similarity_percent:.1f}%, likely mismatched files")
|
|
1066
|
+
if similarity_percent > 90:
|
|
1067
|
+
print(f" WARNING: similarity {similarity_percent:.1f}%, likely undescribed media")
|
|
909
1068
|
|
|
910
|
-
ad_timings = None
|
|
911
1069
|
if stretch_audio:
|
|
912
|
-
|
|
913
|
-
|
|
1070
|
+
# lower memory usage version of np.std for large arrays
|
|
1071
|
+
def low_ram_std(arr):
|
|
1072
|
+
avg = np.mean(arr, dtype=np.float64)
|
|
1073
|
+
return np.sqrt(np.einsum('ij,ij->i', arr, arr, dtype=np.float64)/np.prod(arr.shape) - (avg**2))
|
|
914
1074
|
|
|
915
|
-
|
|
916
|
-
|
|
1075
|
+
# rescale RMS intensity of audio to match video
|
|
1076
|
+
audio_desc_arr *= (low_ram_std(video_arr) / low_ram_std(audio_desc_arr))[:, None]
|
|
917
1077
|
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
smooth_path, ad_detect_sensitivity, boost_sensitivity)
|
|
921
|
-
speech_sample_mask, boost_sample_mask, ad_timings = outputs
|
|
922
|
-
if keep_non_ad:
|
|
923
|
-
video_arr *= speech_sample_mask
|
|
924
|
-
video_arr += video_arr_original * (1 - speech_sample_mask)
|
|
925
|
-
del video_arr_original
|
|
926
|
-
del speech_sample_mask
|
|
927
|
-
else:
|
|
928
|
-
ad_timings = None
|
|
929
|
-
if boost != 0:
|
|
930
|
-
video_arr = video_arr * (1. + (10**(boost / 10.) - 1.) * boost_sample_mask)
|
|
931
|
-
del boost_sample_mask
|
|
1078
|
+
replace_aligned_segments(video_arr, audio_desc_arr, audio_desc_times, video_times, no_pitch_correction)
|
|
1079
|
+
del audio_desc_arr
|
|
932
1080
|
|
|
933
|
-
# prevent peaking by rescaling to within +/-
|
|
1081
|
+
# prevent peaking by rescaling to within +/- 32,766
|
|
934
1082
|
video_arr *= (2**15 - 2.) / np.max(np.abs(video_arr))
|
|
935
1083
|
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
write_replaced_media_to_disk(output_filename, video_arr)
|
|
1084
|
+
print(" processing output file... \r", end='')
|
|
1085
|
+
write_replaced_media_to_disk(output_filename, video_arr, None if has_audio_extension else video_file)
|
|
1086
|
+
del video_arr
|
|
940
1087
|
else:
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
start_key_frame = get_closest_key_frame_time(video_file, video_offset)
|
|
947
|
-
setts_cmd = encode_fit_as_ffmpeg_expr(smooth_path, clips, video_offset, start_key_frame)
|
|
1088
|
+
video_offset = video_times[0] - audio_desc_times[0]
|
|
1089
|
+
# to make ffmpeg cut at the last keyframe before the audio starts, use a timestamp after it
|
|
1090
|
+
after_start_key_frame = get_closest_key_frame_time(video_file, video_offset)
|
|
1091
|
+
print(" processing output file... \r", end='')
|
|
1092
|
+
setts_cmd = encode_fit_as_ffmpeg_expr(audio_desc_times, video_times, video_offset)
|
|
948
1093
|
write_replaced_media_to_disk(output_filename, None, video_file, audio_desc_file,
|
|
949
|
-
setts_cmd,
|
|
1094
|
+
setts_cmd, video_offset, after_start_key_frame)
|
|
950
1095
|
|
|
951
|
-
del video_arr
|
|
952
1096
|
if PLOT_ALIGNMENT_TO_FILE:
|
|
953
1097
|
plot_filename_no_ext = os.path.join(alignment_dir, os.path.splitext(os.path.split(video_file)[1])[0])
|
|
954
|
-
plot_alignment(plot_filename_no_ext, path,
|
|
955
|
-
|
|
1098
|
+
plot_alignment(plot_filename_no_ext, path, audio_desc_times, video_times, similarity_percent,
|
|
1099
|
+
median_slope, stretch_audio, no_pitch_correction)
|
|
1100
|
+
print("All files processed. ")
|
|
956
1101
|
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
1102
|
+
if wx is not None:
|
|
1103
|
+
def write_config_file(config_path, settings):
|
|
1104
|
+
config = configparser.ConfigParser()
|
|
1105
|
+
config.add_section('alignment')
|
|
1106
|
+
config['alignment'] = {}
|
|
1107
|
+
for key, value in settings.items():
|
|
1108
|
+
config['alignment'][key] = str(value)
|
|
1109
|
+
with open(config_path, 'w') as f:
|
|
1110
|
+
config.write(f)
|
|
965
1111
|
|
|
966
|
-
def read_config_file(config_path: Path):
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
1112
|
+
def read_config_file(config_path: Path):
|
|
1113
|
+
config = configparser.ConfigParser()
|
|
1114
|
+
config.read(config_path)
|
|
1115
|
+
settings = {'stretch_audio': config.getboolean('alignment', 'stretch_audio', fallback=False),
|
|
1116
|
+
'prepend': config.get('alignment', 'prepend', fallback='ad_'),
|
|
1117
|
+
'no_pitch_correction': config.getboolean('alignment', 'no_pitch_correction', fallback=False),
|
|
1118
|
+
'output_dir': config.get('alignment', 'output_dir', fallback=default_output_dir),
|
|
1119
|
+
'alignment_dir': config.get('alignment', 'alignment_dir', fallback=default_alignment_dir)}
|
|
1120
|
+
if not config.has_section('alignment'):
|
|
1121
|
+
write_config_file(config_path, settings)
|
|
1122
|
+
return settings
|
|
1123
|
+
|
|
1124
|
+
def set_tooltip(element, tip):
|
|
1125
|
+
element.SetToolTip(tip)
|
|
1126
|
+
# prevent tooltip from disappearing for 30 seconds
|
|
1127
|
+
tooltip_object = element.GetToolTip()
|
|
1128
|
+
if not tooltip_object is None:
|
|
1129
|
+
tooltip_object.SetAutoPop(30000)
|
|
1130
|
+
|
|
1131
|
+
class DialogSettings(wx.Dialog):
|
|
1132
|
+
def __init__(self, parent, config_path, is_dark):
|
|
1133
|
+
wx.Dialog.__init__(self, parent, title="Settings - describealign", size=wx.Size(450,330),
|
|
1134
|
+
style=wx.DEFAULT_DIALOG_STYLE|wx.TAB_TRAVERSAL)
|
|
1135
|
+
# setting the GUI dialog's font causes all contained elements to inherit that font by default
|
|
1136
|
+
self.SetFont(wx.Font(*gui_font))
|
|
1137
|
+
self.SetBackgroundColour(gui_background_color_dark if is_dark else gui_background_color_light)
|
|
1138
|
+
|
|
1139
|
+
self.text_header = wx.StaticText(self, label="Check tooltips (i.e. mouse-over text) for descriptions:")
|
|
1140
|
+
|
|
1141
|
+
self.static_box_sizer_output = wx.StaticBoxSizer(wx.VERTICAL, self, "output_dir")
|
|
1142
|
+
self.dir_picker_output = wx.DirPickerCtrl(self, message="Select a folder", name="output_dir")
|
|
1143
|
+
set_tooltip(self.dir_picker_output, "Directory combined output media is saved to. " + \
|
|
1144
|
+
"Default is \"videos_with_ad\"")
|
|
1145
|
+
|
|
1146
|
+
self.static_box_sizer_alignment = wx.StaticBoxSizer(wx.VERTICAL, self, "alignment_dir")
|
|
1147
|
+
self.dir_picker_alignment = wx.DirPickerCtrl(self, message="Select a folder", name="alignment_dir")
|
|
1148
|
+
set_tooltip(self.dir_picker_alignment, "Directory alignment data and plots are saved to. " + \
|
|
1149
|
+
"Default is \"alignment_plots\"")
|
|
1150
|
+
|
|
1151
|
+
self.text_prepend = wx.StaticText(self, label="prepend:")
|
|
1152
|
+
self.text_ctrl_prepend = wx.TextCtrl(self, name="prepend")
|
|
1153
|
+
set_tooltip(self.text_ctrl_prepend, "Output file name prepend text. Default is \"ad_\"")
|
|
1154
|
+
|
|
1155
|
+
self.checkbox_stretch_audio = wx.CheckBox(self, label="stretch_audio", name="stretch_audio")
|
|
1156
|
+
set_tooltip(self.checkbox_stretch_audio, "Stretches the input audio to fit the input video. " + \
|
|
1157
|
+
"Default is to stretch the video to fit the audio. " + \
|
|
1158
|
+
"Keeps original video audio as secondary tracks. Slower " + \
|
|
1159
|
+
"and uses more RAM when enabled, long videos may cause " + \
|
|
1160
|
+
"paging or Out of Memory errors on low-RAM systems.")
|
|
1161
|
+
self.checkbox_stretch_audio.Bind(wx.EVT_CHECKBOX, self.update_stretch_audio_subsettings)
|
|
1162
|
+
|
|
1163
|
+
self.checkbox_no_pitch_correction = wx.CheckBox(self, label="no_pitch_correction",
|
|
1164
|
+
name="no_pitch_correction")
|
|
1165
|
+
set_tooltip(self.checkbox_no_pitch_correction, "Skips pitch correction step when stretching audio. " + \
|
|
1166
|
+
"Requires --stretch_audio to be set, otherwise " + \
|
|
1167
|
+
"does nothing.")
|
|
1168
|
+
|
|
1169
|
+
self.button_save = wx.Button(self, label="Save")
|
|
1170
|
+
self.button_save.Bind(wx.EVT_BUTTON, self.save_settings)
|
|
1171
|
+
self.button_cancel = wx.Button(self, label="Cancel")
|
|
1172
|
+
self.button_cancel.Bind(wx.EVT_BUTTON, lambda event: self.EndModal(0))
|
|
1173
|
+
|
|
1174
|
+
sizer_dialog = wx.BoxSizer(wx.VERTICAL)
|
|
1175
|
+
sizer_output_dir = wx.BoxSizer(wx.HORIZONTAL)
|
|
1176
|
+
sizer_alignment_dir = wx.BoxSizer(wx.HORIZONTAL)
|
|
1177
|
+
sizer_prepend = wx.BoxSizer(wx.HORIZONTAL)
|
|
1178
|
+
sizer_stretch_audio_no_pitch_correction = wx.BoxSizer(wx.VERTICAL)
|
|
1179
|
+
sizer_save_cancel = wx.BoxSizer(wx.HORIZONTAL)
|
|
1180
|
+
|
|
1181
|
+
# Configure layout with nested Box Sizers:
|
|
1182
|
+
#
|
|
1183
|
+
# Frame
|
|
1184
|
+
# sizer_dialog
|
|
1185
|
+
# text_header
|
|
1186
|
+
# sizer_output_dir
|
|
1187
|
+
# static_box_sizer_output
|
|
1188
|
+
# dir_picker_output
|
|
1189
|
+
# sizer_alignment_dir
|
|
1190
|
+
# static_box_sizer_alignment
|
|
1191
|
+
# dir_picker_alignment
|
|
1192
|
+
# sizer_prepend
|
|
1193
|
+
# text_prepend
|
|
1194
|
+
# text_ctrl_prepend
|
|
1195
|
+
# sizer_stretch_audio_no_pitch_correction
|
|
1196
|
+
# checkbox_stretch_audio
|
|
1197
|
+
# checkbox_no_pitch_correction
|
|
1198
|
+
# sizer_save_cancel
|
|
1199
|
+
# button_save
|
|
1200
|
+
# button_cancel
|
|
1201
|
+
#
|
|
1202
|
+
self.SetSizer(sizer_dialog)
|
|
1203
|
+
sizer_dialog.Add(self.text_header, 0, wx.ALL, 5)
|
|
1204
|
+
sizer_dialog.Add(sizer_output_dir, 1, wx.LEFT|wx.RIGHT|wx.EXPAND, 2)
|
|
1205
|
+
sizer_dialog.Add(sizer_alignment_dir, 1, wx.LEFT|wx.RIGHT|wx.EXPAND, 2)
|
|
1206
|
+
sizer_dialog.Add(sizer_prepend, 1, wx.LEFT|wx.EXPAND, 5)
|
|
1207
|
+
sizer_dialog.Add(sizer_stretch_audio_no_pitch_correction, 1, wx.LEFT|wx.EXPAND, 5)
|
|
1208
|
+
sizer_dialog.Add(sizer_save_cancel, 2, wx.BOTTOM|wx.EXPAND, 5)
|
|
1209
|
+
sizer_prepend.Add(self.text_prepend, 0, wx.ALL|wx.ALIGN_CENTER_VERTICAL, 5)
|
|
1210
|
+
sizer_prepend.Add(self.text_ctrl_prepend, 0, wx.ALIGN_CENTER_VERTICAL, 5)
|
|
1211
|
+
sizer_output_dir.Add(self.static_box_sizer_output, 1, wx.LEFT|wx.RIGHT|wx.ALIGN_CENTER_VERTICAL, 5)
|
|
1212
|
+
self.static_box_sizer_output.Add(self.dir_picker_output, 1, wx.EXPAND)
|
|
1213
|
+
sizer_alignment_dir.Add(self.static_box_sizer_alignment, 1, wx.LEFT|wx.RIGHT|wx.ALIGN_CENTER_VERTICAL, 5)
|
|
1214
|
+
self.static_box_sizer_alignment.Add(self.dir_picker_alignment, 1, wx.EXPAND)
|
|
1215
|
+
sizer_stretch_audio_no_pitch_correction.Add(self.checkbox_stretch_audio, 0, wx.ALL, 5)
|
|
1216
|
+
sizer_stretch_audio_no_pitch_correction.Add(self.checkbox_no_pitch_correction, 0, wx.ALL, 5)
|
|
1217
|
+
sizer_save_cancel.Add((0, 0), 3, wx.EXPAND, 5) # spacer
|
|
1218
|
+
sizer_save_cancel.Add(self.button_save, 0, wx.ALL|wx.ALIGN_CENTER_VERTICAL, 5)
|
|
1219
|
+
sizer_save_cancel.Add((0, 0), 2, wx.EXPAND, 5) # spacer
|
|
1220
|
+
sizer_save_cancel.Add(self.button_cancel, 0, wx.ALL|wx.ALIGN_CENTER_VERTICAL, 5)
|
|
1221
|
+
sizer_save_cancel.Add((0, 0), 3, wx.EXPAND, 5) # spacer
|
|
1222
|
+
|
|
1223
|
+
# centers GUI on the screen
|
|
1224
|
+
self.Centre(wx.BOTH)
|
|
1225
|
+
|
|
1226
|
+
# cache dictionaries mapping setting names to widget setter and getter functions
|
|
1227
|
+
self.setting_getters = {}
|
|
1228
|
+
self.setting_setters = {}
|
|
1229
|
+
for child in self.GetChildren():
|
|
1230
|
+
child_class_name = child.GetClassName()
|
|
1231
|
+
child_name = child.GetName()
|
|
1232
|
+
if child_class_name == "wxDirPickerCtrl":
|
|
1233
|
+
self.setting_getters[child_name] = child.GetPath
|
|
1234
|
+
self.setting_setters[child_name] = child.SetPath
|
|
1235
|
+
if child_class_name in ["wxCheckBox"]:
|
|
1236
|
+
self.setting_getters[child_name] = child.GetValue
|
|
1237
|
+
self.setting_setters[child_name] = child.SetValue
|
|
1238
|
+
if child_class_name in ["wxTextCtrl"]:
|
|
1239
|
+
self.setting_getters[child_name] = child.GetValue
|
|
1240
|
+
self.setting_setters[child_name] = lambda value, child=child: child.SetValue(str(value))
|
|
1241
|
+
self.setting_names = self.setting_getters.keys()
|
|
1242
|
+
|
|
1243
|
+
# initialize setting widgets to saved config values
|
|
1244
|
+
self.config_path = config_path
|
|
1245
|
+
config_file_settings = read_config_file(self.config_path)
|
|
1246
|
+
for setting_name in self.setting_names:
|
|
1247
|
+
self.setting_setters[setting_name](config_file_settings[setting_name])
|
|
983
1248
|
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
[sg.Column([[sg.Text('output_dir:', size=(10, 1.2), pad=(1,5)),
|
|
995
|
-
sg.Input(default_text=str(settings['output_dir']), size=(22, 1.2), pad=(10,5), key='output_dir',
|
|
996
|
-
tooltip='Directory combined output media is saved to. Default is "videos_with_ad"'),
|
|
997
|
-
sg.FolderBrowse(button_text="Browse Folder", key='output_browse')]])],
|
|
998
|
-
[sg.Column([[sg.Text('alignment_dir:', size=(13, 1.2), pad=(1,5)),
|
|
999
|
-
sg.Input(default_text=str(settings['alignment_dir']), size=(22, 1.2), pad=(10,5), key='alignment_dir',
|
|
1000
|
-
tooltip='Directory alignment data and plots are saved to. Default is "alignment_plots"'),
|
|
1001
|
-
sg.FolderBrowse(button_text="Browse Folder", key='alignment_browse')]], pad=(2,7))],
|
|
1002
|
-
[sg.Column([[sg.Text('smoothness:', size=(12, 1), pad=(1,5)),
|
|
1003
|
-
sg.Input(default_text=str(settings['smoothness']), size=(8, 1.2), pad=(10,5), key='smoothness',
|
|
1004
|
-
tooltip='Lower values make the alignment more accurate when there are skips ' + \
|
|
1005
|
-
'(e.g. describer pauses), but also make it more likely to misalign. ' + \
|
|
1006
|
-
'Default is 50.')]])],
|
|
1007
|
-
[sg.Checkbox('stretch_audio', default=settings['stretch_audio'], key='stretch_audio', change_submits=True,
|
|
1008
|
-
tooltip='Stretches the input audio to fit the input video. ' + \
|
|
1009
|
-
'Default is to stretch the video to fit the audio.')],
|
|
1010
|
-
[sg.Checkbox('keep_non_ad', default=settings['keep_non_ad'], key='keep_non_ad',
|
|
1011
|
-
disabled=not settings['stretch_audio'],
|
|
1012
|
-
tooltip='Tries to only replace segments with audio description. Useful if ' + \
|
|
1013
|
-
'video\'s audio quality is better. Default is to replace all aligned audio. ' + \
|
|
1014
|
-
'Requires --stretch_audio to be set, otherwise does nothing.')],
|
|
1015
|
-
[sg.Column([[sg.Text('boost:', size=(6, 1), pad=(1,5)),
|
|
1016
|
-
sg.Input(default_text=str(settings['boost']), size=(8, 1.2), pad=(10,5),
|
|
1017
|
-
key='boost', disabled=not settings['stretch_audio'],
|
|
1018
|
-
tooltip='Boost (or quieten) description volume. Units are decibels (dB), so ' + \
|
|
1019
|
-
'-3 makes the describer about 2x quieter, while 3 makes them 2x louder. ' + \
|
|
1020
|
-
'Requires --stretch_audio to be set, otherwise does nothing.')]])],
|
|
1021
|
-
[sg.Column([[sg.Text('ad_detect_sensitivity:', size=(21, 1.2), pad=(2,5)),
|
|
1022
|
-
sg.Input(default_text=str(settings['ad_detect_sensitivity']), size=(8, 1.2), pad=(10,5),
|
|
1023
|
-
key='ad_detect_sensitivity', disabled=not settings['stretch_audio'],
|
|
1024
|
-
tooltip='Audio description detection sensitivity ratio. Higher values make ' + \
|
|
1025
|
-
'--keep_non_ad more likely to replace aligned audio. Default is 0.6')]])],
|
|
1026
|
-
[sg.Column([[sg.Text('boost_sensitivity:', size=(17, 1.2), pad=(1,5)),
|
|
1027
|
-
sg.Input(default_text=str(settings['boost_sensitivity']), size=(8, 1.2), pad=(10,5),
|
|
1028
|
-
key='boost_sensitivity', disabled=not settings['stretch_audio'],
|
|
1029
|
-
tooltip='Higher values make --boost less likely to miss a description, but ' + \
|
|
1030
|
-
'also make it more likely to boost non-description audio. Default is 0.4')]])],
|
|
1031
|
-
[sg.Checkbox('no_pitch_correction', default=settings['no_pitch_correction'], key='no_pitch_correction',
|
|
1032
|
-
disabled=not settings['stretch_audio'],
|
|
1033
|
-
tooltip='Skips pitch correction step when stretching audio. ' + \
|
|
1034
|
-
'Requires --stretch_audio to be set, otherwise does nothing.')],
|
|
1035
|
-
[sg.Column([[sg.Submit('Save', pad=(40,3)),
|
|
1036
|
-
sg.Button('Cancel')]], pad=((135,3),10))]]
|
|
1037
|
-
settings_window = sg.Window('Settings - describealign', layout, font=('Arial', 16), finalize=True)
|
|
1038
|
-
settings_window['extension'].set_focus()
|
|
1039
|
-
while True:
|
|
1040
|
-
event, values = settings_window.read()
|
|
1041
|
-
if event in (sg.WIN_CLOSED, 'Cancel') or settings_window.TKrootDestroyed:
|
|
1042
|
-
break
|
|
1043
|
-
if event == 'stretch_audio':
|
|
1044
|
-
# work around bug in PySimpleGUIWx's InputText Update function where enabling/disabling are flipped
|
|
1045
|
-
if IS_RUNNING_WINDOWS:
|
|
1046
|
-
settings_window['boost'].Update(disabled = values['stretch_audio'])
|
|
1047
|
-
settings_window['ad_detect_sensitivity'].Update(disabled = values['stretch_audio'])
|
|
1048
|
-
settings_window['boost_sensitivity'].Update(disabled = values['stretch_audio'])
|
|
1249
|
+
# initialize stretch_audio subsettings to be disabled/enabled
|
|
1250
|
+
self.update_stretch_audio_subsettings()
|
|
1251
|
+
|
|
1252
|
+
set_background_color(self, is_dark)
|
|
1253
|
+
|
|
1254
|
+
def update_stretch_audio_subsettings(self, event=None):
|
|
1255
|
+
subsettings = [self.checkbox_no_pitch_correction]
|
|
1256
|
+
if self.checkbox_stretch_audio.IsChecked():
|
|
1257
|
+
for subsetting in subsettings:
|
|
1258
|
+
subsetting.Enable()
|
|
1049
1259
|
else:
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1260
|
+
for subsetting in subsettings:
|
|
1261
|
+
subsetting.Disable()
|
|
1262
|
+
|
|
1263
|
+
def save_settings(self, event):
|
|
1264
|
+
settings = {}
|
|
1265
|
+
for setting_name in self.setting_names:
|
|
1266
|
+
settings[setting_name] = self.setting_getters[setting_name]()
|
|
1267
|
+
write_config_file(self.config_path, settings)
|
|
1268
|
+
self.EndModal(0)
|
|
1269
|
+
|
|
1270
|
+
class QueueWriter(io.TextIOWrapper):
|
|
1271
|
+
def __init__(self, queue) -> None:
|
|
1272
|
+
super().__init__(buffer=io.BytesIO())
|
|
1273
|
+
self._queue = queue
|
|
1274
|
+
|
|
1275
|
+
def write(self, s: str) -> int:
|
|
1276
|
+
self._queue.put(s)
|
|
1277
|
+
return len(s)
|
|
1062
1278
|
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1279
|
+
def combine_print_exceptions(print_queue, *args, **kwargs):
|
|
1280
|
+
writer = QueueWriter(print_queue)
|
|
1281
|
+
with redirect_stdout(writer), redirect_stderr(writer):
|
|
1282
|
+
try:
|
|
1283
|
+
combine(*args, **kwargs)
|
|
1284
|
+
except Exception:
|
|
1285
|
+
print(" ERROR: exception raised")
|
|
1286
|
+
traceback.print_exc()
|
|
1287
|
+
|
|
1288
|
+
class FrameCombine(wx.Frame):
|
|
1289
|
+
def __init__(self, parent, config_path, video_files, audio_files, is_dark):
|
|
1290
|
+
wx.Frame.__init__(self, parent, title="Combining - describealign", size=wx.Size(800,600))
|
|
1291
|
+
# setting the GUI frame's font causes all contained elements to inherit that font by default
|
|
1292
|
+
self.SetFont(wx.Font(*gui_font))
|
|
1293
|
+
self.SetBackgroundColour(gui_background_color_dark if is_dark else gui_background_color_light)
|
|
1294
|
+
# wrap all widgets within a panel to enable tab traversal (i.e. pressing tab to swap GUI focus)
|
|
1295
|
+
self.panel0 = wx.Panel(self, style=wx.TAB_TRAVERSAL)
|
|
1296
|
+
|
|
1297
|
+
self.text_ctrl_output = wx.TextCtrl(self.panel0, style=wx.TE_MULTILINE|wx.TE_READONLY|wx.TE_RICH)
|
|
1298
|
+
|
|
1299
|
+
self.button_close = wx.Button(self.panel0, label="Close")
|
|
1300
|
+
self.button_close.Bind(wx.EVT_BUTTON, self.attempt_close)
|
|
1301
|
+
# also capture other close events such as alt+f4 or clicking the X in the top corner of the frame
|
|
1302
|
+
self.Bind(wx.EVT_CLOSE, self.attempt_close)
|
|
1303
|
+
|
|
1304
|
+
self.update_timer = wx.Timer(self)
|
|
1305
|
+
self.Bind(wx.EVT_TIMER, self.update_gui, self.update_timer)
|
|
1306
|
+
|
|
1307
|
+
sizer_panel_outer = wx.BoxSizer(wx.VERTICAL)
|
|
1308
|
+
sizer_panel_inner = wx.BoxSizer(wx.VERTICAL)
|
|
1309
|
+
sizer_close_button = wx.BoxSizer(wx.HORIZONTAL)
|
|
1310
|
+
|
|
1311
|
+
# Configure layout with nested Box Sizers:
|
|
1312
|
+
#
|
|
1313
|
+
# Frame
|
|
1314
|
+
# sizer_panel_outer
|
|
1315
|
+
# panel0
|
|
1316
|
+
# sizer_panel_inner
|
|
1317
|
+
# text_ctrl_output
|
|
1318
|
+
# sizer_close_button
|
|
1319
|
+
# button_close
|
|
1320
|
+
#
|
|
1321
|
+
self.SetSizer(sizer_panel_outer)
|
|
1322
|
+
sizer_panel_outer.Add(self.panel0, 1, wx.EXPAND|wx.ALL, 5)
|
|
1323
|
+
self.panel0.SetSizer(sizer_panel_inner)
|
|
1324
|
+
sizer_panel_inner.Add(self.text_ctrl_output, 1, wx.ALL|wx.EXPAND, 5)
|
|
1325
|
+
sizer_panel_inner.Add(sizer_close_button, 0, wx.EXPAND, 5)
|
|
1326
|
+
sizer_close_button.Add((0, 0), 1, wx.EXPAND, 5) # spacer
|
|
1327
|
+
sizer_close_button.Add(self.button_close, 0, wx.ALL, 5)
|
|
1328
|
+
sizer_close_button.Add((0, 0), 1, wx.EXPAND, 5) # spacer
|
|
1329
|
+
|
|
1330
|
+
# centers GUI on the screen
|
|
1331
|
+
self.Centre(wx.BOTH)
|
|
1332
|
+
|
|
1333
|
+
set_background_color(self, is_dark)
|
|
1334
|
+
|
|
1335
|
+
self.config_path = config_path
|
|
1336
|
+
self.overwrite_last_line = False
|
|
1337
|
+
self.display_line('Combining media files:')
|
|
1338
|
+
self.text_ctrl_output.SetInsertionPoint(0)
|
|
1339
|
+
|
|
1340
|
+
# launch combiner using settings from config file, redirecting its output to a queue
|
|
1341
|
+
self.print_queue = multiprocessing.Queue()
|
|
1342
|
+
settings = read_config_file(self.config_path)
|
|
1343
|
+
settings.update({'yes':True})
|
|
1344
|
+
self.combine_process = multiprocessing.Process(target=combine_print_exceptions,
|
|
1345
|
+
args=(self.print_queue, video_files, audio_files),
|
|
1346
|
+
kwargs=settings, daemon=True)
|
|
1347
|
+
self.combine_process.start()
|
|
1348
|
+
self.update_gui()
|
|
1349
|
+
|
|
1350
|
+
def attempt_close(self, event):
|
|
1351
|
+
if self.combine_process.is_alive():
|
|
1352
|
+
dialog = wx.MessageDialog(self, "Warning: combiner is still running, stop it and close anyway?",
|
|
1353
|
+
"Warning", wx.YES_NO|wx.ICON_WARNING)
|
|
1354
|
+
response = dialog.ShowModal()
|
|
1355
|
+
if (response == wx.ID_YES):
|
|
1356
|
+
self.combine_process.terminate()
|
|
1357
|
+
self.Destroy()
|
|
1358
|
+
elif (response == wx.ID_NO):
|
|
1359
|
+
# If the EVT_CLOSE came from the OS, let the OS know it didn't succeed
|
|
1360
|
+
if event.GetEventType() == wx.EVT_CLOSE.evtType[0]:
|
|
1361
|
+
event.Veto(True)
|
|
1362
|
+
else:
|
|
1363
|
+
self.Destroy()
|
|
1364
|
+
|
|
1365
|
+
def set_last_line_color(self, color, line_start):
|
|
1366
|
+
num_lines = self.text_ctrl_output.GetNumberOfLines()
|
|
1367
|
+
end = self.text_ctrl_output.GetLastPosition()
|
|
1368
|
+
self.text_ctrl_output.SetStyle(line_start, end, wx.TextAttr("black", color))
|
|
1067
1369
|
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1370
|
+
def display_line(self, line):
|
|
1371
|
+
if self.overwrite_last_line:
|
|
1372
|
+
# skip the empty line following lines ending in "\r"
|
|
1373
|
+
if line == "":
|
|
1374
|
+
return
|
|
1375
|
+
num_lines = self.text_ctrl_output.GetNumberOfLines()
|
|
1376
|
+
start = self.text_ctrl_output.XYToPosition(0,num_lines-2)
|
|
1377
|
+
end = self.text_ctrl_output.GetLastPosition()
|
|
1378
|
+
self.text_ctrl_output.Remove(start, end)
|
|
1379
|
+
self.overwrite_last_line = False
|
|
1380
|
+
if line[-1:] == "\r":
|
|
1381
|
+
self.overwrite_last_line = True
|
|
1382
|
+
line = line[:-1].rstrip(' ') + "\r"
|
|
1383
|
+
line_start = self.text_ctrl_output.GetLastPosition()
|
|
1384
|
+
self.text_ctrl_output.AppendText(line)
|
|
1385
|
+
# highlight warnings by changing their background color to light orange
|
|
1386
|
+
if line[:10] == " WARNING:":
|
|
1387
|
+
self.set_last_line_color(wx.Colour(255, 188, 64), line_start)
|
|
1388
|
+
# highlight errors by changing their background color to red
|
|
1389
|
+
if line[:8] == " ERROR:":
|
|
1390
|
+
self.set_last_line_color(wx.Colour(255, 128, 128), line_start)
|
|
1391
|
+
|
|
1392
|
+
def update_gui(self, event=None):
|
|
1393
|
+
lines = []
|
|
1394
|
+
while not self.print_queue.empty():
|
|
1395
|
+
lines.append(self.print_queue.get())
|
|
1396
|
+
if len(lines) > 0:
|
|
1397
|
+
cursor_position = self.text_ctrl_output.GetInsertionPoint()
|
|
1398
|
+
self.text_ctrl_output.Freeze()
|
|
1399
|
+
for line in lines:
|
|
1400
|
+
self.display_line(line)
|
|
1401
|
+
self.text_ctrl_output.SetInsertionPoint(cursor_position)
|
|
1402
|
+
self.text_ctrl_output.Thaw()
|
|
1403
|
+
self.update_timer.StartOnce(gui_update_interval_ms)
|
|
1071
1404
|
|
|
1072
|
-
def
|
|
1073
|
-
|
|
1074
|
-
|
|
1405
|
+
def migrate_config(old_path: Optional[Path], new_path: Path) -> None:
|
|
1406
|
+
"""
|
|
1407
|
+
Migrate configuration from old location.
|
|
1408
|
+
|
|
1409
|
+
Only runs if the old_path exists but new_path does not
|
|
1410
|
+
"""
|
|
1411
|
+
if new_path.exists() or not old_path or not old_path.exists():
|
|
1412
|
+
return
|
|
1413
|
+
|
|
1414
|
+
old_data = old_path.read_text(encoding='utf-8')
|
|
1415
|
+
new_path.write_text(old_data, encoding='utf-8')
|
|
1416
|
+
print(f"Configuration migrated to {new_path}")
|
|
1075
1417
|
try:
|
|
1076
|
-
|
|
1077
|
-
except
|
|
1078
|
-
traceback.
|
|
1418
|
+
old_path.unlink()
|
|
1419
|
+
except OSError as exc:
|
|
1420
|
+
print("Failed to remove old config:", *traceback.format_exception_only(exc))
|
|
1421
|
+
else:
|
|
1422
|
+
print("Successfully removed old config file.")
|
|
1079
1423
|
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
if
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
if IS_RUNNING_WINDOWS:
|
|
1104
|
-
output_textbox.WxTextCtrl.SetInsertionPoint(cursor_position)
|
|
1105
|
-
event, values = combine_window.read(timeout=100)
|
|
1106
|
-
# window closed event isn't always emitted, so also manually check window status
|
|
1107
|
-
if event == sg.WIN_CLOSED or combine_window.TKrootDestroyed:
|
|
1108
|
-
if proc.is_alive():
|
|
1109
|
-
proc.terminate()
|
|
1110
|
-
break
|
|
1111
|
-
if event == 'Close':
|
|
1112
|
-
if not proc.is_alive():
|
|
1113
|
-
combine_window.DisableClose = False
|
|
1114
|
-
break
|
|
1115
|
-
selection = sg.PopupYesNo('Combiner is still running, stop it and close anyway?')
|
|
1116
|
-
if selection != 'Yes':
|
|
1117
|
-
continue
|
|
1118
|
-
proc.terminate()
|
|
1119
|
-
combine_window.DisableClose = False
|
|
1120
|
-
break
|
|
1121
|
-
combine_window.close()
|
|
1424
|
+
class ListCtrlDropTarget(wx.FileDropTarget):
|
|
1425
|
+
def __init__(self, list_ctrl, parent_frame):
|
|
1426
|
+
super().__init__()
|
|
1427
|
+
self.list_ctrl = list_ctrl
|
|
1428
|
+
self.parent_frame = parent_frame
|
|
1429
|
+
|
|
1430
|
+
def expand_folders(self, files):
|
|
1431
|
+
expanded_files = []
|
|
1432
|
+
for file in files:
|
|
1433
|
+
if os.path.isdir(file):
|
|
1434
|
+
for dir, subdirs, dir_files in os.walk(file):
|
|
1435
|
+
for dir_file in dir_files:
|
|
1436
|
+
expanded_files.append(os.path.join(dir, dir_file))
|
|
1437
|
+
else:
|
|
1438
|
+
expanded_files.append(file)
|
|
1439
|
+
return expanded_files
|
|
1440
|
+
|
|
1441
|
+
def OnDropFiles(self, x, y, files):
|
|
1442
|
+
files = self.expand_folders(files)
|
|
1443
|
+
valid_file_types = self.parent_frame.list_ctrl_file_types_drop[self.list_ctrl]
|
|
1444
|
+
files = [file for file in files if os.path.splitext(file)[-1][1:] in valid_file_types]
|
|
1445
|
+
self.parent_frame.populate_list_ctrl(self.list_ctrl, natsort.os_sorted(files))
|
|
1446
|
+
return True
|
|
1122
1447
|
|
|
1123
|
-
def
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
Only runs if the old_path exists but new_path does not
|
|
1128
|
-
"""
|
|
1129
|
-
if new_path.exists() or not old_path or not old_path.exists():
|
|
1130
|
-
return
|
|
1131
|
-
|
|
1132
|
-
old_data = old_path.read_text(encoding='utf-8')
|
|
1133
|
-
new_path.write_text(old_data, encoding='utf-8')
|
|
1134
|
-
print(f"Configuration migrated to {new_path}")
|
|
1135
|
-
try:
|
|
1136
|
-
old_path.unlink()
|
|
1137
|
-
except OSError as exc:
|
|
1138
|
-
print("Failed to remove old config:", *traceback.format_exception_only(exc))
|
|
1139
|
-
else:
|
|
1140
|
-
print("Successfully removed old config file.")
|
|
1448
|
+
def get_children(window):
|
|
1449
|
+
children = list(window.GetChildren())
|
|
1450
|
+
subchildren = [subchild for child in children for subchild in get_children(child)]
|
|
1451
|
+
return children + subchildren
|
|
1141
1452
|
|
|
1142
|
-
def
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1453
|
+
def set_background_color(window, is_dark):
|
|
1454
|
+
children = get_children(window)
|
|
1455
|
+
for window in children + [window]:
|
|
1456
|
+
if is_dark:
|
|
1457
|
+
if isinstance(window, (wx.ListCtrl, wx.Button, wx.TextCtrl)):
|
|
1458
|
+
window.SetBackgroundColour("Black")
|
|
1459
|
+
else:
|
|
1460
|
+
window.SetBackgroundColour(gui_background_color_dark)
|
|
1461
|
+
window.SetForegroundColour("White" if is_dark else "Black")
|
|
1462
|
+
|
|
1463
|
+
class FrameMain(wx.Frame):
|
|
1464
|
+
def __init__(self, parent):
|
|
1465
|
+
wx.Frame.__init__(self, parent, title="describealign", size=wx.Size(800, 500))
|
|
1466
|
+
# setting the GUI frame's font causes all contained elements to inherit that font by default
|
|
1467
|
+
self.SetFont(wx.Font(*gui_font))
|
|
1468
|
+
appearance = wx.SystemSettings.GetAppearance()
|
|
1469
|
+
self.is_dark = appearance.IsDark() or appearance.IsUsingDarkBackground()
|
|
1470
|
+
self.SetBackgroundColour(gui_background_color_dark if self.is_dark else gui_background_color_light)
|
|
1471
|
+
|
|
1472
|
+
# wrap all widgets within a panel to enable tab traversal (i.e. pressing tab to swap GUI focus)
|
|
1473
|
+
self.panel0 = wx.Panel(self, style=wx.TAB_TRAVERSAL)
|
|
1474
|
+
|
|
1475
|
+
self.text_header = wx.StaticText(self.panel0, label="Select media files to combine:")
|
|
1476
|
+
self.text_header.SetFont(self.text_header.GetFont().Scale(1.7))
|
|
1477
|
+
|
|
1478
|
+
# Video Input selection and display row of GUI
|
|
1479
|
+
self.static_box_sizer_video = wx.StaticBoxSizer(wx.HORIZONTAL, self.panel0, "Video Input")
|
|
1480
|
+
self.list_ctrl_video = self.init_list_ctrl(self.static_box_sizer_video.GetStaticBox(),
|
|
1481
|
+
"Drag and Drop Videos Here or Press Browse Video")
|
|
1482
|
+
set_tooltip(self.list_ctrl_video, "Video filenames are listed here in the sorted order they will " + \
|
|
1483
|
+
"be used as input. Drag and Drop or press Browse to overwrite.")
|
|
1484
|
+
self.button_browse_video = wx.Button(self.static_box_sizer_video.GetStaticBox(), label="Browse Video")
|
|
1485
|
+
set_tooltip(self.button_browse_video, "Select one or more video files as input.")
|
|
1486
|
+
self.button_browse_video.Bind(wx.EVT_BUTTON, lambda event: self.browse_files(self.list_ctrl_video))
|
|
1487
|
+
|
|
1488
|
+
# Audio Input selection and display row of GUI
|
|
1489
|
+
self.static_box_sizer_audio = wx.StaticBoxSizer(wx.HORIZONTAL, self.panel0, "Audio Input")
|
|
1490
|
+
self.list_ctrl_audio = self.init_list_ctrl(self.static_box_sizer_audio.GetStaticBox(),
|
|
1491
|
+
"Drag and Drop Audio Here or Press Browse Audio")
|
|
1492
|
+
set_tooltip(self.list_ctrl_audio, "Audio filenames are listed here in the sorted order they will " + \
|
|
1493
|
+
"be used as input. Drag and Drop or press Browse to overwrite.")
|
|
1494
|
+
self.button_browse_audio = wx.Button(self.static_box_sizer_audio.GetStaticBox(), label="Browse Audio")
|
|
1495
|
+
set_tooltip(self.button_browse_audio, "Select one or more audio files as input.")
|
|
1496
|
+
self.button_browse_audio.Bind(wx.EVT_BUTTON, lambda event: self.browse_files(self.list_ctrl_audio))
|
|
1497
|
+
|
|
1498
|
+
self.button_combine = wx.Button(self.panel0, label="Combine")
|
|
1499
|
+
set_tooltip(self.button_combine, "Combine selected video and audio files.")
|
|
1500
|
+
self.button_combine.Bind(wx.EVT_BUTTON, self.open_combine)
|
|
1501
|
+
self.button_settings = wx.Button(self.panel0, label="Settings")
|
|
1502
|
+
set_tooltip(self.button_settings, "Edit settings for the GUI and algorithm.")
|
|
1503
|
+
self.button_settings.Bind(wx.EVT_BUTTON, self.open_settings)
|
|
1504
|
+
|
|
1505
|
+
sizer_panel_outer = wx.BoxSizer(wx.VERTICAL)
|
|
1506
|
+
sizer_panel_inner = wx.BoxSizer(wx.VERTICAL)
|
|
1507
|
+
sizer_header = wx.BoxSizer(wx.HORIZONTAL)
|
|
1508
|
+
sizer_video = wx.BoxSizer(wx.HORIZONTAL)
|
|
1509
|
+
sizer_audio = wx.BoxSizer(wx.HORIZONTAL)
|
|
1510
|
+
sizer_combine_settings = wx.BoxSizer(wx.HORIZONTAL)
|
|
1511
|
+
|
|
1512
|
+
# Configure layout with nested Box Sizers:
|
|
1513
|
+
#
|
|
1514
|
+
# Frame
|
|
1515
|
+
# sizer_panel_outer
|
|
1516
|
+
# panel0
|
|
1517
|
+
# sizer_panel_inner
|
|
1518
|
+
# sizer_header
|
|
1519
|
+
# text_header
|
|
1520
|
+
# sizer_video
|
|
1521
|
+
# list_ctrl_video
|
|
1522
|
+
# button_browse_video
|
|
1523
|
+
# sizer_audio
|
|
1524
|
+
# list_ctrl_audio
|
|
1525
|
+
# button_browse_audio
|
|
1526
|
+
# sizer_combine_settings
|
|
1527
|
+
# button_combine
|
|
1528
|
+
# button_settings
|
|
1529
|
+
#
|
|
1530
|
+
self.SetSizer(sizer_panel_outer)
|
|
1531
|
+
sizer_panel_outer.Add(self.panel0, 1, wx.EXPAND|wx.ALL, 5)
|
|
1532
|
+
self.panel0.SetSizer(sizer_panel_inner)
|
|
1533
|
+
sizer_panel_inner.Add(sizer_header, 3, wx.EXPAND, 5)
|
|
1534
|
+
sizer_panel_inner.Add(sizer_video, 9, wx.EXPAND, 5)
|
|
1535
|
+
sizer_panel_inner.Add(sizer_audio, 9, wx.TOP|wx.EXPAND, 3)
|
|
1536
|
+
sizer_panel_inner.Add(sizer_combine_settings, 3, wx.EXPAND, 5)
|
|
1537
|
+
sizer_header.Add(self.text_header, 0, wx.ALL|wx.ALIGN_CENTER_VERTICAL, 5)
|
|
1538
|
+
sizer_video.Add(self.static_box_sizer_video, 1, wx.LEFT|wx.RIGHT|wx.EXPAND, 3)
|
|
1539
|
+
self.static_box_sizer_video.Add(self.list_ctrl_video, 1, wx.BOTTOM|wx.EXPAND, 2)
|
|
1540
|
+
self.static_box_sizer_video.Add(self.button_browse_video, 0,
|
|
1541
|
+
wx.LEFT|wx.BOTTOM|wx.RIGHT|wx.ALIGN_CENTER_VERTICAL, 10)
|
|
1542
|
+
sizer_audio.Add(self.static_box_sizer_audio, 1, wx.LEFT|wx.RIGHT|wx.EXPAND, 3)
|
|
1543
|
+
self.static_box_sizer_audio.Add(self.list_ctrl_audio, 1, wx.BOTTOM|wx.EXPAND, 2)
|
|
1544
|
+
self.static_box_sizer_audio.Add(self.button_browse_audio, 0,
|
|
1545
|
+
wx.LEFT|wx.BOTTOM|wx.RIGHT|wx.ALIGN_CENTER_VERTICAL, 10)
|
|
1546
|
+
sizer_combine_settings.Add((0, 0), 7, wx.EXPAND, 5) # spacer
|
|
1547
|
+
sizer_combine_settings.Add(self.button_combine, 0, wx.ALL|wx.ALIGN_CENTER_VERTICAL, 5)
|
|
1548
|
+
sizer_combine_settings.Add((0, 0), 2, wx.EXPAND, 5) # spacer
|
|
1549
|
+
sizer_combine_settings.Add(self.button_settings, 0, wx.ALL|wx.ALIGN_CENTER_VERTICAL, 5)
|
|
1550
|
+
sizer_combine_settings.Add((0, 0), 7, wx.EXPAND, 5) # spacer
|
|
1551
|
+
|
|
1552
|
+
# centers GUI on the screen
|
|
1553
|
+
self.Centre(wx.BOTH)
|
|
1554
|
+
|
|
1555
|
+
all_video_file_types = [('All Video File Types', '*.' + ';*.'.join(VIDEO_EXTENSIONS)),]
|
|
1556
|
+
all_audio_file_types = [('All Audio File Types', '*.' + ';*.'.join(AUDIO_EXTENSIONS)),]
|
|
1557
|
+
all_video_and_audio_file_types = [('All Video and Audio File Types',
|
|
1558
|
+
'*.' + ';*.'.join(VIDEO_EXTENSIONS | AUDIO_EXTENSIONS)),]
|
|
1559
|
+
self.video_file_types = [(ext, f"*.{ext}") for ext in VIDEO_EXTENSIONS]
|
|
1560
|
+
self.audio_file_types = [(ext, f"*.{ext}") for ext in AUDIO_EXTENSIONS]
|
|
1561
|
+
self.video_and_audio_file_types = self.video_file_types + self.audio_file_types
|
|
1562
|
+
self.video_file_types = all_video_file_types + self.video_file_types
|
|
1563
|
+
self.audio_file_types = all_audio_file_types + self.audio_file_types
|
|
1564
|
+
self.video_and_audio_file_types = all_video_file_types + all_video_and_audio_file_types + \
|
|
1565
|
+
self.video_and_audio_file_types
|
|
1566
|
+
self.video_file_types = '|'.join([f'{type[0]} ({type[1]})|{type[1]}' for type in self.video_file_types])
|
|
1567
|
+
self.audio_file_types = '|'.join([f'{type[0]} ({type[1]})|{type[1]}' for type in self.audio_file_types])
|
|
1568
|
+
self.video_and_audio_file_types = '|'.join([f'{type[0]} ({type[1]})|{type[1]}' for type \
|
|
1569
|
+
in self.video_and_audio_file_types])
|
|
1570
|
+
|
|
1571
|
+
# track the allowed file types and selected files' full paths for each List Ctrl
|
|
1572
|
+
self.list_ctrl_file_types_browse = {self.list_ctrl_video: self.video_and_audio_file_types,
|
|
1573
|
+
self.list_ctrl_audio: self.audio_file_types}
|
|
1574
|
+
self.list_ctrl_file_types_drop = {self.list_ctrl_video: self.video_file_types,
|
|
1575
|
+
self.list_ctrl_audio: self.audio_file_types}
|
|
1576
|
+
self.list_ctrl_files_selected = {self.list_ctrl_video: [],
|
|
1577
|
+
self.list_ctrl_audio: []}
|
|
1578
|
+
|
|
1579
|
+
self.config_path = self.get_config()
|
|
1580
|
+
|
|
1581
|
+
set_background_color(self, self.is_dark)
|
|
1582
|
+
|
|
1583
|
+
def init_list_ctrl(self, parent_panel, default_text):
|
|
1584
|
+
list_ctrl = wx.ListCtrl(parent_panel, style=wx.LC_NO_HEADER|wx.LC_REPORT|wx.BORDER_SUNKEN|wx.HSCROLL)
|
|
1585
|
+
list_ctrl.EnableSystemTheme(False) # get rid of vertical grid lines on Windows
|
|
1586
|
+
list_ctrl.SetMinSize(wx.Size(-1,80))
|
|
1587
|
+
list_ctrl.SetDropTarget(ListCtrlDropTarget(list_ctrl, self))
|
|
1588
|
+
list_ctrl.InsertColumn(0, "")
|
|
1589
|
+
list_ctrl.InsertItem(0, default_text)
|
|
1590
|
+
list_ctrl.SetColumnWidth(0, wx.LIST_AUTOSIZE)
|
|
1591
|
+
list_ctrl.Bind(wx.EVT_CHAR, self.delete_from_list_ctrl)
|
|
1592
|
+
return list_ctrl
|
|
1593
|
+
|
|
1594
|
+
def populate_list_ctrl(self, list_ctrl, files):
|
|
1595
|
+
self.list_ctrl_files_selected[list_ctrl] = files
|
|
1596
|
+
if len(files) == 0:
|
|
1597
|
+
files = ["No files with valid file types found"]
|
|
1598
|
+
list_ctrl.DeleteAllItems()
|
|
1599
|
+
list_ctrl.DeleteAllColumns()
|
|
1600
|
+
list_ctrl.InsertColumn(0, "")
|
|
1601
|
+
for i, file in enumerate(files):
|
|
1602
|
+
list_ctrl.InsertItem(i, os.path.basename(file))
|
|
1603
|
+
list_ctrl.SetColumnWidth(0, wx.LIST_AUTOSIZE)
|
|
1604
|
+
|
|
1605
|
+
def browse_files(self, list_ctrl):
|
|
1606
|
+
dialog = wx.FileDialog(self, wildcard=self.list_ctrl_file_types_browse[list_ctrl], style=wx.FD_MULTIPLE)
|
|
1607
|
+
if dialog.ShowModal() == wx.ID_OK:
|
|
1608
|
+
files = dialog.GetPaths()
|
|
1609
|
+
self.populate_list_ctrl(list_ctrl, files)
|
|
1610
|
+
|
|
1611
|
+
def delete_from_list_ctrl(self, event):
|
|
1612
|
+
if event.GetKeyCode() == wx.WXK_DELETE:
|
|
1613
|
+
list_ctrl = event.GetEventObject()
|
|
1614
|
+
item_index = list_ctrl.GetFirstSelected()
|
|
1615
|
+
if item_index == -1:
|
|
1616
|
+
item_index = list_ctrl.GetFocusedItem()
|
|
1617
|
+
items_to_delete = []
|
|
1618
|
+
while item_index != -1:
|
|
1619
|
+
items_to_delete.append(item_index)
|
|
1620
|
+
item_index = list_ctrl.GetNextSelected(item_index)
|
|
1621
|
+
for item_index in items_to_delete[::-1]:
|
|
1622
|
+
if len(self.list_ctrl_files_selected[list_ctrl]) != 0:
|
|
1623
|
+
list_ctrl.DeleteItem(item_index)
|
|
1624
|
+
del self.list_ctrl_files_selected[list_ctrl][item_index]
|
|
1625
|
+
else:
|
|
1626
|
+
event.Skip()
|
|
1627
|
+
|
|
1628
|
+
def open_combine(self, event):
|
|
1629
|
+
video_files = self.list_ctrl_files_selected[self.list_ctrl_video]
|
|
1630
|
+
audio_files = self.list_ctrl_files_selected[self.list_ctrl_audio]
|
|
1631
|
+
if len(video_files) == 0:
|
|
1632
|
+
error_dialog = wx.MessageDialog(self, "Error: no video input selected.", "Error", wx.OK|wx.ICON_ERROR)
|
|
1633
|
+
error_dialog.ShowModal()
|
|
1634
|
+
elif len(audio_files) == 0:
|
|
1635
|
+
error_dialog = wx.MessageDialog(self, "Error: no audio input selected.", "Error", wx.OK|wx.ICON_ERROR)
|
|
1636
|
+
error_dialog.ShowModal()
|
|
1637
|
+
elif len(video_files) != len(audio_files):
|
|
1638
|
+
error_dialog = wx.MessageDialog(self, f"Error: different numbers of video ({len(video_files)}) " + \
|
|
1639
|
+
f"and audio ({len(audio_files)}) inputs.",
|
|
1640
|
+
"Error", wx.OK|wx.ICON_ERROR)
|
|
1641
|
+
error_dialog.ShowModal()
|
|
1642
|
+
else:
|
|
1643
|
+
frame_combine = FrameCombine(None, self.config_path, video_files, audio_files, self.is_dark)
|
|
1644
|
+
self.list_ctrl_video.SetFocus()
|
|
1645
|
+
frame_combine.Show()
|
|
1646
|
+
|
|
1647
|
+
def open_settings(self, event):
|
|
1648
|
+
dialog_settings = DialogSettings(None, self.config_path, self.is_dark)
|
|
1649
|
+
dialog_settings.ShowModal()
|
|
1650
|
+
dialog_settings.Destroy()
|
|
1651
|
+
|
|
1652
|
+
def get_config(self):
|
|
1653
|
+
config_path = platformdirs.user_config_path(appname='describealign', appauthor=False,
|
|
1654
|
+
ensure_exists=True) / 'config.ini'
|
|
1655
|
+
old_paths = [
|
|
1656
|
+
# Place in chronological order (oldest -> newest)
|
|
1657
|
+
Path(__file__).resolve().parent / 'config.ini',
|
|
1658
|
+
platformdirs.user_config_path(appname='describealign', ensure_exists=True) / 'config.ini',
|
|
1659
|
+
]
|
|
1660
|
+
# Get newest existent path
|
|
1661
|
+
old_config = next((file for file in reversed(old_paths) if file.exists()), None,)
|
|
1662
|
+
try:
|
|
1663
|
+
migrate_config(old_config, config_path)
|
|
1664
|
+
except OSError as exc:
|
|
1665
|
+
print(f"Error migrating old config:", *traceback.format_exception_only(exc))
|
|
1666
|
+
print(f"Old config left in place at {old_config}")
|
|
1667
|
+
return config_path
|
|
1668
|
+
|
|
1669
|
+
def get_version_hash(filename):
|
|
1160
1670
|
try:
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
filetype_sep = ';' if IS_RUNNING_WINDOWS else ' '
|
|
1169
|
-
all_audio_file_types = [('All Audio File Types', '*.' + f'{filetype_sep}*.'.join(AUDIO_EXTENSIONS)),]
|
|
1170
|
-
all_video_file_types = [('All Video File Types', '*.' + f'{filetype_sep}*.'.join(VIDEO_EXTENSIONS)),]
|
|
1171
|
-
all_video_and_audio_file_types = [('All Video and Audio File Types',
|
|
1172
|
-
'*.' + f'{filetype_sep}*.'.join(VIDEO_EXTENSIONS | AUDIO_EXTENSIONS)),]
|
|
1173
|
-
audio_file_types = [(ext, f"*.{ext}") for ext in AUDIO_EXTENSIONS]
|
|
1174
|
-
video_and_audio_file_types = [(ext, f"*.{ext}") for ext in VIDEO_EXTENSIONS] + audio_file_types
|
|
1175
|
-
audio_file_types = all_audio_file_types + audio_file_types
|
|
1176
|
-
video_and_audio_file_types = all_video_file_types + all_video_and_audio_file_types + video_and_audio_file_types
|
|
1177
|
-
# work around bug in PySimpleGUIWx's convert_tkinter_filetypes_to_wx function
|
|
1178
|
-
if IS_RUNNING_WINDOWS:
|
|
1179
|
-
file_fix = lambda file_types: file_types[:1] + [(f'|{type[0]}', type[1]) for type in file_types[1:]]
|
|
1180
|
-
audio_file_types = file_fix(audio_file_types)
|
|
1181
|
-
video_and_audio_file_types = file_fix(video_and_audio_file_types)
|
|
1182
|
-
|
|
1183
|
-
layout = [[sg.Text('Select media files to combine:', size=(40, 2), font=('Arial', 20), pad=(3,15))],
|
|
1184
|
-
[sg.Column([[sg.Text('Video Input:', size=(11, 2), pad=(1,5)),
|
|
1185
|
-
sg.Input(size=(35, 1.2), pad=(10,5), key='-VIDEO_FILES-',
|
|
1186
|
-
tooltip='List video filenames here, in order, separated by semicolons'),
|
|
1187
|
-
sg.FilesBrowse(button_text="Browse Video",
|
|
1188
|
-
file_types=video_and_audio_file_types,
|
|
1189
|
-
tooltip='Select one or more video files')]], pad=(2,7))],
|
|
1190
|
-
[sg.Column([[sg.Text('Audio Input:', size=(11, 2), pad=(1,5)),
|
|
1191
|
-
sg.Input(size=(35, 1.2), pad=(10,5), key='-AUDIO_FILES-',
|
|
1192
|
-
tooltip='List audio filenames here, in order, separated by semicolons'),
|
|
1193
|
-
sg.FilesBrowse(button_text="Browse Audio",
|
|
1194
|
-
file_types=audio_file_types,
|
|
1195
|
-
tooltip='Select one or more audio files')]], pad=(2,7))],
|
|
1196
|
-
[sg.Column([[sg.Submit('Combine', pad=(40,3), tooltip='Combine selected video and audio files'),
|
|
1197
|
-
sg.Button('Settings', tooltip='Edit settings for the GUI and algorithm.')]],
|
|
1198
|
-
pad=((135,3),10))]]
|
|
1199
|
-
window = sg.Window('describealign', layout, font=('Arial', 16), resizable=False, finalize=True)
|
|
1200
|
-
window['-VIDEO_FILES-'].set_focus()
|
|
1201
|
-
while True:
|
|
1202
|
-
event, values = window.read()
|
|
1203
|
-
if event == 'Combine':
|
|
1204
|
-
if len(values['-VIDEO_FILES-']) == 0 or \
|
|
1205
|
-
len(values['-AUDIO_FILES-']) == 0:
|
|
1206
|
-
window.disable()
|
|
1207
|
-
sg.Popup('Error: empty input field.', font=('Arial', 20))
|
|
1208
|
-
window.enable()
|
|
1209
|
-
continue
|
|
1210
|
-
video_files = values['-VIDEO_FILES-'].split(';')
|
|
1211
|
-
if len(video_files) == 1:
|
|
1212
|
-
video_files = video_files[0]
|
|
1213
|
-
audio_files = values['-AUDIO_FILES-'].split(';')
|
|
1214
|
-
if len(audio_files) == 1:
|
|
1215
|
-
audio_files = audio_files[0]
|
|
1216
|
-
combine_gui(video_files, audio_files, config_path)
|
|
1217
|
-
if event == 'Settings':
|
|
1218
|
-
window.disable()
|
|
1219
|
-
settings_gui(config_path)
|
|
1220
|
-
window.enable()
|
|
1221
|
-
if event == sg.WIN_CLOSED:
|
|
1222
|
-
break
|
|
1223
|
-
window.close()
|
|
1671
|
+
with open(filename, 'rb') as f:
|
|
1672
|
+
data = f.read()
|
|
1673
|
+
sha_hash = hashlib.sha1(data).hexdigest()
|
|
1674
|
+
return sha_hash[:8]
|
|
1675
|
+
except:
|
|
1676
|
+
return "None"
|
|
1224
1677
|
|
|
1225
1678
|
# Entry point for command line interaction, for example:
|
|
1226
1679
|
# > describealign video.mp4 audio_desc.mp3
|
|
1227
1680
|
def command_line_interface():
|
|
1228
1681
|
if len(sys.argv) < 2:
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1682
|
+
if wx is not None:
|
|
1683
|
+
# No args, run gui
|
|
1684
|
+
print('No input arguments detected, starting GUI...')
|
|
1685
|
+
# the following line is necessary on MacOS X to fix the filectrlpicker
|
|
1686
|
+
# https://docs.wxpython.org/wx.FileDialog.html#wx-filedialog
|
|
1687
|
+
# https://github.com/wxWidgets/Phoenix/issues/2368
|
|
1688
|
+
if platform.system() == 'Darwin':
|
|
1689
|
+
wx.SystemOptions.SetOption('osx.openfiledialog.always-show-types', 1)
|
|
1690
|
+
app = wx.App()
|
|
1691
|
+
main_gui = FrameMain(None)
|
|
1692
|
+
main_gui.Show()
|
|
1693
|
+
app.MainLoop()
|
|
1694
|
+
sys.exit(0)
|
|
1695
|
+
else:
|
|
1696
|
+
print("Can't launch GUI and arguments missing.\nGUI dependencies missing.")
|
|
1233
1697
|
|
|
1234
|
-
parser = argparse.ArgumentParser(
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
parser.add_argument("audio", help='An audio file or directory containing audio files.',
|
|
1239
|
-
|
|
1240
|
-
help='Lower values make the alignment more accurate when there are skips ' + \
|
|
1241
|
-
'(e.g. describer pauses), but also make it more likely to misalign. ' + \
|
|
1242
|
-
'Default is 50.')
|
|
1698
|
+
parser = argparse.ArgumentParser(description="Replaces a video's sound with an audio description.",
|
|
1699
|
+
usage="describealign video_file.mp4 audio_file.mp3")
|
|
1700
|
+
parser.add_argument("video", help='A video file or directory containing video files.',
|
|
1701
|
+
nargs='?', default=None)
|
|
1702
|
+
parser.add_argument("audio", help='An audio file or directory containing audio files.',
|
|
1703
|
+
nargs='?', default=None)
|
|
1243
1704
|
parser.add_argument('--stretch_audio', action='store_true',
|
|
1244
1705
|
help='Stretches the input audio to fit the input video. ' + \
|
|
1245
|
-
'Default is to stretch the video to fit the audio.'
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
'
|
|
1249
|
-
'Requires --stretch_audio to be set, otherwise does nothing.')
|
|
1250
|
-
parser.add_argument('--boost', type=float, default=0,
|
|
1251
|
-
help='Boost (or quieten) description volume. Units are decibels (dB), so ' + \
|
|
1252
|
-
'-3 makes the describer about 2x quieter, while 3 makes them 2x louder. ' + \
|
|
1253
|
-
'Requires --stretch_audio to be set, otherwise does nothing.')
|
|
1254
|
-
parser.add_argument('--ad_detect_sensitivity', type=float, default=.6,
|
|
1255
|
-
help='Audio description detection sensitivity ratio. Higher values make ' + \
|
|
1256
|
-
'--keep_non_ad more likely to replace aligned audio. Default is 0.6')
|
|
1257
|
-
parser.add_argument('--boost_sensitivity', type=float, default=.4,
|
|
1258
|
-
help='Higher values make --boost less likely to miss a description, but ' + \
|
|
1259
|
-
'also make it more likely to boost non-description audio. Default is 0.4')
|
|
1706
|
+
'Default is to stretch the video to fit the audio. ' + \
|
|
1707
|
+
'Keeps original video audio as secondary tracks. Slower ' + \
|
|
1708
|
+
'and uses more RAM when enabled, long videos may cause ' + \
|
|
1709
|
+
'paging or Out of Memory errors on low-RAM systems.')
|
|
1260
1710
|
parser.add_argument('--yes', action='store_true',
|
|
1261
1711
|
help='Auto-skips user prompts asking to verify information.')
|
|
1262
1712
|
parser.add_argument("--prepend", default="ad_", help='Output file name prepend text. Default is "ad_"')
|
|
@@ -1267,23 +1717,42 @@ def command_line_interface():
|
|
|
1267
1717
|
help='Directory combined output media is saved to. Default is "videos_with_ad"')
|
|
1268
1718
|
parser.add_argument("--alignment_dir", default=default_alignment_dir,
|
|
1269
1719
|
help='Directory alignment data and plots are saved to. Default is "alignment_plots"')
|
|
1270
|
-
parser.add_argument("--extension", default="copy",
|
|
1271
|
-
help='File type of output video (e.g. mkv). When set to "copy", copies the ' + \
|
|
1272
|
-
'file type of the corresponding input video. Default is "copy".')
|
|
1273
1720
|
parser.add_argument("--install-ffmpeg", action="store_true",
|
|
1274
|
-
help="Install the required ffmpeg binaries and then exit. This is meant to be" + \
|
|
1721
|
+
help="Install the required ffmpeg binaries and then exit. This is meant to be " + \
|
|
1275
1722
|
"run from a privileged installer process (e.g. OS X Installer)")
|
|
1723
|
+
parser.add_argument('--version', action='store_true',
|
|
1724
|
+
help='Checks and prints the installed version of describealign.')
|
|
1276
1725
|
args = parser.parse_args()
|
|
1277
1726
|
|
|
1278
|
-
if args.
|
|
1727
|
+
if args.version:
|
|
1728
|
+
import importlib
|
|
1729
|
+
cur_dir = os.getcwd()
|
|
1730
|
+
if sys.path[0] == cur_dir:
|
|
1731
|
+
# ignore describealign.py in current directory
|
|
1732
|
+
del sys.path[0]
|
|
1733
|
+
installed_spec = importlib.util.find_spec('describealign')
|
|
1734
|
+
sys.path = [cur_dir] + sys.path
|
|
1735
|
+
else:
|
|
1736
|
+
installed_spec = importlib.util.find_spec('describealign')
|
|
1737
|
+
if installed_spec is None:
|
|
1738
|
+
print("describealign is not installed")
|
|
1739
|
+
else:
|
|
1740
|
+
installed_path = os.path.abspath(installed_spec.origin)
|
|
1741
|
+
this_script_path = os.path.abspath(__file__)
|
|
1742
|
+
if installed_path != this_script_path:
|
|
1743
|
+
print("WARNING: describealign is not being run from the installed version")
|
|
1744
|
+
print(f" installed path: {installed_path}")
|
|
1745
|
+
print(f" content hash: {get_version_hash(installed_path)}")
|
|
1746
|
+
print(f" this file path: {this_script_path}")
|
|
1747
|
+
print(f" content hash: {get_version_hash(this_script_path)}")
|
|
1748
|
+
print(f"installed version: {importlib.metadata.version('describealign')}")
|
|
1749
|
+
elif args.install_ffmpeg:
|
|
1279
1750
|
# Make sure the file is world executable
|
|
1280
1751
|
os.chmod(get_ffmpeg(), 0o755)
|
|
1281
1752
|
os.chmod(get_ffprobe(), 0o755)
|
|
1282
|
-
elif args.video
|
|
1283
|
-
combine(args.video, args.audio, args.
|
|
1284
|
-
args.
|
|
1285
|
-
args.prepend, args.no_pitch_correction, args.output_dir, args.alignment_dir,
|
|
1286
|
-
args.extension)
|
|
1753
|
+
elif args.video and args.audio:
|
|
1754
|
+
combine(args.video, args.audio, args.stretch_audio, args.yes, args.prepend,
|
|
1755
|
+
args.no_pitch_correction, args.output_dir, args.alignment_dir)
|
|
1287
1756
|
else:
|
|
1288
1757
|
parser.print_usage()
|
|
1289
1758
|
|