tictacsync 0.5a0__py3-none-any.whl → 0.7a0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
tictacsync/yaltc.py CHANGED
@@ -5,7 +5,7 @@ import lmfit, tempfile
5
5
  # from skimage.morphology import closing, square
6
6
  from numpy import arcsin, sin, pi
7
7
  from matplotlib.lines import Line2D
8
- import math, re, os, sys
8
+ import math, re, os, sys, itertools
9
9
  import sox
10
10
  from subprocess import Popen, PIPE
11
11
  from pathlib import Path
@@ -17,7 +17,8 @@ logging.config.dictConfig({
17
17
  from datetime import datetime, timezone, timedelta
18
18
  from collections import deque
19
19
  from loguru import logger
20
- from sklearn.mixture import GaussianMixture
20
+ from skimage.morphology import closing, erosion, remove_small_objects
21
+ from skimage.measure import regionprops, label
21
22
  import ffmpeg, shutil
22
23
  from rich import print
23
24
  from rich.console import Console
@@ -28,6 +29,8 @@ try:
28
29
  except:
29
30
  import device_scanner
30
31
 
32
+ TEENSY_MAX_LAG = 128/44100 # sec, duration of a default length audio block
33
+
31
34
 
32
35
  CACHING = True
33
36
  DEL_TEMP = False
@@ -41,6 +44,13 @@ WORDWIDTHFACTOR = 2
41
44
 
42
45
  OVER_NOISE_SYNC_DETECT_LEVEL = 2
43
46
 
47
+ ################## pasted from FSKfreqCalculator.py output:
48
+ F1 = 630.00 # Hertz
49
+ F2 = 1190.00 # Hz , both from FSKfreqCalculator output
50
+ SYMBOL_LENGTH = 14.286 # ms, from FSKfreqCalculator.py
51
+ N_SYMBOLS = 35 # including sync pulse
52
+ ##################
53
+
44
54
  MINIMUM_LENGTH = 4 # sec
45
55
  TRIAL_TIMES = [ # in seconds
46
56
  (0.5, -2),
@@ -52,18 +62,12 @@ TRIAL_TIMES = [ # in seconds
52
62
  (3.5, -2),
53
63
  (3.5, -3.5),
54
64
  ]
55
- SOUND_EXTRACT_LENGTH = 1.56 # second
65
+ SOUND_EXTRACT_LENGTH = (10*SYMBOL_LENGTH*1e-3 + 1) # second
56
66
  SYMBOL_LENGTH_TOLERANCE = 0.07 # relative
57
67
  FSK_TOLERANCE = 60 # Hz
58
68
  SAMD21_LATENCY = 63 # microseconds, for DAC conversion
59
69
  YEAR_ZERO = 2021
60
70
 
61
- ################## pasted from FSKfreqCalculator.py output:
62
- F1 = 630.00 # Hertz
63
- F2 = 1190.00 # Hz , both from FSKfreqCalculator output
64
- SYMBOL_LENGTH = 14.286 # ms, from FSKfreqCalculator.py
65
- N_SYMBOLS_SAMD21 = 35 # including sync pulse
66
- ##################
67
71
 
68
72
  BPF_LOW_FRQ, BPF_HIGH_FRQ = (0.5*F1, 2*F2)
69
73
 
@@ -81,6 +85,12 @@ def _pathname(tempfile_or_path):
81
85
  tempfile_or_path,
82
86
  type(tempfile_or_path)))
83
87
 
88
+ # for skimage.measure.regionprops
89
+ def _width(region):
90
+ _,x1,_,x2 = region.bbox
91
+ return x2-x1
92
+
93
+
84
94
  def to_precision(x,p):
85
95
  """
86
96
  returns a string representation of x formatted with a precision of p
@@ -154,8 +164,8 @@ class Decoder:
154
164
  rec : Recording
155
165
  recording on which the decoder is working
156
166
 
157
- SN_ratio : float
158
- signal over noise ratio in dB.
167
+ effective_word_duration : float
168
+ duration of a word, influenced by ucontroller clock
159
169
 
160
170
  pulse_detection_level : float
161
171
  level used to detect sync pulse
@@ -176,7 +186,7 @@ class Decoder:
176
186
 
177
187
  """
178
188
 
179
- def __init__(self, aRec):
189
+ def __init__(self, aRec, do_plots):
180
190
  """
181
191
  Initialises Decoder
182
192
 
@@ -186,587 +196,307 @@ class Decoder:
186
196
 
187
197
  """
188
198
  self.rec = aRec
199
+ self.do_plots = do_plots
189
200
  self.clear_decoder()
190
201
 
202
+
191
203
  def clear_decoder(self):
192
204
  self.sound_data_extract = None
193
- self.cached_convolution_fit = {'sound_extract_position': None}
194
205
  self.pulse_detection_level = None
195
- self.silent_zone_indices = None
196
206
  self.detected_pulse_position = None
197
- self.estimated_pulse_position = None
198
-
199
- def set_sound_extract_and_sr(self, extract, s_r, where):
200
- self.sound_extract = extract
201
- self.samplerate = s_r
202
- self.sound_extract_position = where
203
- self.cached_convolution_fit = {'sound_extract_position': None}
204
- logger.debug('sound_extract set, samplerate %i location %i'%(s_r, where))
205
-
206
-
207
- # there's always at least one complete 0.5 silence interval in a 1.5 second signal
208
-
209
- def _get_envelope(self):
210
- """
211
- Compute self.sound_extract envelope, filtering its hilbert transform.
212
- Uses scipy.signal.hilbert() and scipy.signal.savgol_filter();
213
- window_length and polyorder savgol_filter() parameters values
214
- have been found by hit and miss on audio data.
215
-
216
- maybe:
217
- Values are roughly normalized: between 0 and approximately 1.0
218
-
219
- Returns
220
- -------
221
- numpy.ndarray the same length of self.recording.sound_extract
222
-
223
- """
224
- WINDOW_LENGTH, POLYORDER = (15, 3) # parameters found by experiment, hit and miss
225
- absolute_of_hilbert = np.abs(scipy.signal.hilbert(self.sound_extract))
226
- envelope = scipy.signal.savgol_filter(absolute_of_hilbert,
227
- WINDOW_LENGTH, POLYORDER)
228
- logger.debug('self.sound_extract envelope length %i samples'%len(envelope))
229
- return envelope
230
-
231
- def _get_signal_level(self):
232
- abs_signal = abs(self.sound_extract)
233
- return 2 * abs_signal.mean() # since 50% duty cycle
234
-
235
- def _get_approx_pulse_position(self):
236
- """
237
- Returns the estimated pulse position using the detected silent
238
- zone . The position is in samples number
239
- relative to extract beginning
240
- """
241
- # if self.detected_pulse_position:
242
- # logger.debug('returning detected value')
243
- # return self.detected_pulse_position
244
- if self.estimated_pulse_position:
245
- logger.debug('returning cached estimated value')
246
- return self.estimated_pulse_position ###############################
247
- _, silence_center_x = self._fit_triangular_signal_to_convoluted_env()
248
- # symbol_width_samples = 1e-3*SYMBOL_LENGTH
249
- self.estimated_pulse_position = silence_center_x + int(0.5*(
250
- 0.5 - 1e-3*SYMBOL_LENGTH)*self.samplerate)
251
- logger.debug('returning estimated value from silence mid position')
252
- return self.estimated_pulse_position
253
-
254
- def _get_pulse_position(self):
255
- # relative to extract beginning
256
- if self.detected_pulse_position:
257
- logger.debug('returning detected value')
258
- return self.detected_pulse_position
259
- return None
260
-
261
- def _get_pulse_detection_level(self):
262
- # return level at which the sync pulse will be detected
263
- if self.pulse_detection_level is None:
264
- silence_floor = self._get_silence_floor()
265
- # lower_BFSK_level = silence_floor
266
- # pulse_position = self._get_pulse_position()
267
- lower_BFSK_level = self._get_minimal_bfsk()
268
- value = math.sqrt(silence_floor * lower_BFSK_level)
269
- # value = OVER_NOISE_SYNC_DETECT_LEVEL * silence_floor
270
- logger.debug('setting pulse_detection_level to %f'%value)
271
- self.pulse_detection_level = value
272
- return value
273
- else:
274
- return self.pulse_detection_level
275
-
276
- def _get_square_convolution(self):
277
- """
278
- Compute self.sound_extract envelope convolution with a square signal of
279
- 0.5 second (+ SYMBOL_LENGTH) width, using numpy.convolve().
280
- Values are roughly normalized: between 0 and approximately 1.0
281
-
282
-
283
- Returns:
284
-
285
- #1 the normalized convolution, a numpy.ndarray shorter than
286
- self.sound_extract, see numpy.convolve(..., mode='valid')
287
-
288
- #2 a list of int, samples indexes where the convolution is computed
289
-
290
- """
291
- sqr_window_width = int((0.5 + SYMBOL_LENGTH/1e3)*self.samplerate) # in samples
292
- sqr_signal = np.ones(sqr_window_width,dtype=int)
293
- envelope = self._get_envelope()
294
- mean = envelope.mean()
295
- if mean: # in case of zero padding (? dont remember why)
296
- factor = 0.5/mean # since 50% duty cycle
297
- else:
298
- factor = 1
299
- normalized_envelope = factor*envelope
300
- convol = np.convolve(normalized_envelope, sqr_signal,
301
- mode='valid')/sqr_window_width
302
- start = int(sqr_window_width/2)
303
-
304
- x = range(start, len(convol) + start)
305
- return [*x], convol
306
-
307
- def _get_word_envelope(self):
308
- """
309
- Chop the signal envelope keeping the word region and smooth it over the
310
- longest BFSK period
311
- """
312
- SR = self.samplerate
313
- envelope = self._get_envelope()
314
- pulse_position = self._get_approx_pulse_position()
315
- samples_to_end = len(self.sound_extract) - pulse_position
316
- is_too_near_the_end = samples_to_end/SR < 0.5
317
- logger.debug('pulse_position is_too_near_the_end %s'%
318
- is_too_near_the_end)
319
- if is_too_near_the_end:
320
- pulse_position -= SR # one second sooner
321
- symbol_width_samples = 1e-3*SYMBOL_LENGTH*SR
322
- word_start = int(pulse_position + 3*symbol_width_samples)
323
- word_end = int(pulse_position + 0.5*SR)
324
- word_end -= int(2*symbol_width_samples) # slide to the left a little
325
- logger.debug('word start, end: %i %i (in extract)'%(
326
- word_start, word_end))
327
- logger.debug('word start, end: %i %i (in file)'%(
328
- word_start + self.sound_extract_position,
329
- word_end + self.sound_extract_position))
330
- w_envelope = envelope[word_start : word_end]
331
- word_envelope_truncated = word_end-word_start != len(w_envelope)
332
- logger.debug('w_envelope is sliced out of bounds: %s'%(
333
- str(word_envelope_truncated)))
334
- logger.debug('word envelope length %i samples %f secs'%(
335
- len(w_envelope), len(w_envelope)/SR))
336
- max_period = int(self.samplerate*max(1/F1,1/F2))
337
- logger.debug('max BFSK period %i in samples'%max_period)
338
- period_window = np.ones(max_period,dtype=int)/max_period
339
- # smooth over longest BFSK period
340
- return np.convolve(w_envelope, period_window, mode='same')
341
-
342
- def _get_minimal_bfsk(self):
343
- """
344
- because of non-flat frequency response, bfsk bits dont have the same
345
- amplitude. This returns the least of both by detecting a bimodal
346
- gaussian distribution
347
-
348
- """
349
- # w_envelope = self._get_word_envelope()
350
- # word_start = int(min_position + shift + 0.3*self.samplerate)
351
- # word = w_envelope[word_start : int(word_start + 0.4*self.samplerate)]
352
- word = self._get_word_envelope()
353
- # plt.plot(word)
354
- # plt.show()
355
- n = len(word)
356
- word = word.reshape(n, 1)
357
- gm = GaussianMixture(n_components=2, random_state=0).fit(word)
358
- bfsk_minimal_amplitude = min(gm.means_)
359
- logger.debug('bfsk_minimal_amplitude %f'%bfsk_minimal_amplitude)
360
- return bfsk_minimal_amplitude
361
-
362
- def _fit_triangular_signal_to_convoluted_env(self):
207
+
208
+ def set_sound_extract_and_sr(self, sound_extract, samplerate, sound_extract_position):
363
209
  """
364
- Try to fit a triangular signal to the envelope convoluted with a square
365
- signal to evaluate if audio is composed of 0.5 sec signal and 0.5 s
366
- silence. If so, the convolution is a triangular fct and a
367
- Levenberg–Marquardt fit is used to detect this occurence (lmfit).
368
- Results are cached in self.cached_convolution_fit alongside
369
- self.sound_extract_position for hit/miss checks.
370
-
210
+ Sets:
211
+ self.sound_extract -- mono data of short duration
212
+ self.samplerate -- in Hz
213
+ self.sound_extract_position -- position in the whole file
371
214
 
372
- Returns
373
- -------
374
- float
375
- chi_sqr from lmfit.minimize(), indicative of fit quality
376
- int
377
- position of triangular minimum (base of the v shape), this
378
- corresponds to the center of silent zone.
215
+ Computes and sets:
216
+ self.pulse_detection_level
217
+ self.sound_extract_one_bit
218
+ self.words_props (contains the sync pulse too)
379
219
 
220
+ Returns nothing
380
221
  """
381
- cached_value = self.cached_convolution_fit['sound_extract_position']
382
- logger.debug('cache hit? asked:%s cached: %s'%(
383
- self.sound_extract_position,
384
- cached_value))
385
- logger.debug('(sound_extract_position values)')
386
- if (CACHING and
387
- # cache_is_clean and
388
- cached_value and
389
- math.isclose(
390
- self.sound_extract_position,
391
- cached_value,
392
- abs_tol=0.1)):
393
- logger.debug('yes, fit values cached:')
394
- v1 = self.cached_convolution_fit['chi_square']
395
- v2 = self.cached_convolution_fit['minimum position']
396
- v2_file = v2 + self.sound_extract_position
397
- logger.debug('cached chi_sq: %s minimum position in file: %s'%
398
- (v1, v2_file))
399
- return (v1, v2) ####################################################
400
- # cached!
401
- x_shifted, convolution = self._get_square_convolution()
402
- # see numpy.convolve(..., mode='valid')
403
- x = np.arange(len(convolution))
404
- trig_params = lmfit.Parameters()
405
- trig_params.add(
406
- 'A', value=1, min=0, max=2
407
- )
408
- period0 = 2*self.samplerate
409
- trig_params.add(
410
- 'period', value=period0, min=0.9*period0,
411
- max=1.1*period0
412
- )
413
- trig_params.add(
414
- 'min_position', value=len(convolution)/2
415
- ) # at center
416
- def trig_wave(pars, x, signal_data=None):
417
- # looking for phase sx with a sin of 1 sec period and 0<y<1.0
418
- A = pars['A']
419
- p = pars['period']
420
- mp = pars['min_position']
421
- model = 2*A*arcsin(abs(sin((x - mp)*2*pi/p)))/pi
422
- if signal_data is None:
423
- return model ###################################################
424
- return model - signal_data
425
- fit_trig = lmfit.minimize(
426
- trig_wave, trig_params,
427
- args=(x,), kws={'signal_data': convolution}
428
- )
429
- chi_square = fit_trig.chisqr
430
- shift = x_shifted[0] # convolution is shorter than sound envelope
431
- min_position = int(fit_trig.params['min_position'].value) + shift
432
- logger.debug('chi_square %.1f minimum convolution position %i in file'%
433
- (chi_square, min_position + self.sound_extract_position))
434
- self.cached_convolution_fit['sound_extract_position'] = \
435
- self.sound_extract_position
436
- self.cached_convolution_fit['chi_square'] = chi_square
437
- self.cached_convolution_fit['minimum position'] = min_position
438
-
439
- return chi_square, min_position + shift
222
+ logger.debug('sound_extract: %s, samplerate: %s Hz, sound_extract_position %s'%(
223
+ sound_extract, samplerate, sound_extract_position))
224
+ if len(sound_extract) == 0:
225
+ logger.error('sound extract is empty, is sound track duration OK?')
226
+ raise Exception('sound extract is empty, is sound track duration OK?')
227
+ self.sound_extract_position = sound_extract_position
228
+ self.samplerate = samplerate
229
+ self.sound_extract = sound_extract
230
+ self.pulse_detection_level = np.std(sound_extract)/4
231
+ logger.debug('pulse_detection_level %f'%self.pulse_detection_level)
232
+ bits = np.abs(sound_extract)>self.pulse_detection_level
233
+ N_ones = round(1.5*SYMBOL_LENGTH*1e-3*samplerate) # so it includes sync pulse
234
+ self.sound_extract_one_bit = closing(bits, np.ones(N_ones))
235
+ if self.do_plots:
236
+ self._plot_extract()
237
+ logger.debug('sound_extract_one_bit len %i'%len(self.sound_extract_one_bit))
238
+ self.words_props = regionprops(label(np.array(2*[self.sound_extract_one_bit]))) # new
440
239
 
441
240
  def extract_seems_TicTacCode(self):
442
- """margin
443
- evaluate if sound data is half signal, half silence
444
- no test is done on frequency components nor BFSK modulation.
445
-
446
- Returns
447
- -------
448
- True if sound seems TicTacCode
449
-
450
- """
451
- chi_square, _ = self._fit_triangular_signal_to_convoluted_env()
452
- seems_TicTacCode = chi_square < 200 # good fit so, yes
453
- logger.debug('seems TicTacCode: %s'%seems_TicTacCode)
454
- return seems_TicTacCode
241
+ """
242
+ Determines if signal in sound_extract seems to be TTC.
455
243
 
456
- def _get_silent_zone_indices(self):
457
- """
458
- Returns silent zone boundary positions relative to the start
459
- of self.sound_extract.
244
+ Uses the conditions below:
460
245
 
461
- Returns
462
- -------
463
- left_window_boundary : int
464
- left indice.
465
- right_window_boundary : int
466
- right indice.
246
+ Extract duration is 1.143 s.
247
+ In self.word_props (list of morphology.regionprops):
248
+ if one region, duration should be in [0.499 0.512] sec
249
+ if two regions, total duration should be in [0.50 0.655]
467
250
 
251
+ Returns True if self.sound_data_extract seems TicTacCode
468
252
  """
469
- if self.silent_zone_indices:
470
- return self.silent_zone_indices ####################################
471
- _, silence_center_position = \
472
- self._fit_triangular_signal_to_convoluted_env()
473
- srate = self.samplerate
474
- half_window = int(SAFE_SILENCE_WINDOW_WIDTH * 1e-3 * srate/2)
475
- left_window_boundary = silence_center_position - half_window
476
- right_window_boundary = silence_center_position + half_window
477
- # margin = 0.75 * srate
478
- values = np.array([left_window_boundary, right_window_boundary,
479
- silence_center_position])
480
- values += self.sound_extract_position # samples pos in file
481
- logger.debug('silent zone, left: %i, right %i, center %i'%tuple(values))
482
- self.silent_zone_indices = (left_window_boundary, right_window_boundary)
483
- return self.silent_zone_indices
484
-
485
- def _get_silence_floor(self):
486
- # analyses the 0.5 silence zone
487
- start_silent_zone, end_silent_zone = self._get_silent_zone_indices()
488
- signal = self.sound_extract
489
- silent_signal = signal[start_silent_zone:end_silent_zone]
490
- max_value = 1.001*np.abs(silent_signal).max() # a little headroom
491
- max_value = 0 # should toggle this with a CLI option
492
- five_sigmas = 5 * silent_signal.std()
493
- return max(max_value, five_sigmas) # if guassian, five sigmas will do it
494
-
495
- def make_silence_analysis_plot(self, title=None, filename=None):
496
- # save figure in filename if set, otherwise
497
- # start an interactive plot, title is for matplotlib
498
- pulse_pos_in_file = self._get_approx_pulse_position()
499
- pulse_pos_in_file += self.sound_extract_position
500
- pulse_position_sec = pulse_pos_in_file/self.samplerate
501
- duration_in_sec = self.rec.get_duration()
502
- if pulse_position_sec > duration_in_sec/2:
503
- pulse_position_sec -= duration_in_sec
504
- title = 'Silence analysis around %.2f s'%(pulse_position_sec)
505
- logger.debug('make_silence_analysis_plot(title=%s, filename=%s)'%(
506
- title, filename))
507
- start_silent_zone, end_silent_zone = self._get_silent_zone_indices()
508
- signal = self.sound_extract
509
- x_signal = range(len(signal))
510
- x_convolution, convolution = self._get_square_convolution()
511
- scaled_convo = self._get_signal_level()*convolution
512
- # since 0 < convolution < 1
513
- trig_level = self._get_pulse_detection_level()
514
- sound_extract_position = self.sound_extract_position
515
- def x2f(nx):
516
- return nx + sound_extract_position
517
- def f2x(nf):
518
- return nf - sound_extract_position
253
+ failing_comment = '' # used as a flag
254
+ props = self.words_props
255
+ if len(props) not in [1,2]:
256
+ failing_comment = 'len(props) not in [1,2]: %i'%len(props)
257
+ if len(props) == 1:
258
+ w = _width(props[0])/self.samplerate
259
+ # self.effective_word_duration = w
260
+ # logger.debug('effective_word_duration %f (one region)'%w)
261
+ if not 0.499 < w < 0.512: # TODO: move as TOP OF FILE PARAMS
262
+ failing_comment = '_width %f not in [0.499 0.512]'%w
263
+ else: # 2 regions
264
+ widths = [_width(p)/self.samplerate for p in props] # in sec
265
+ total_w = sum(widths)
266
+ # extra_window_duration = SOUND_EXTRACT_LENGTH - 1
267
+ # eff_w = total_w - extra_window_duration
268
+ # logger.debug('effective_word_duration %f (two regions)'%eff_w)
269
+ if not 0.5 < total_w < 0.655:
270
+ failing_comment = 'two regions duration %f not in [0.50 0.655]\n%s'%(total_w, widths)
271
+ # fig, ax = plt.subplots()
272
+ # p(ax, sound_extract_one_bit)
273
+ logger.debug('failing_comment: %s'%(
274
+ 'none' if failing_comment=='' else failing_comment))
275
+ return failing_comment == '' # no comment = extract seems TicTacCode
276
+
277
+ def _plot_extract(self):
519
278
  fig, ax = plt.subplots()
520
- plt.title(title)
521
- secax = ax.secondary_xaxis('top', functions=(x2f, f2x))
522
- # secax.set_xlabel('position in file')
523
- # ax.set_xlabel('position in extract')
524
- yt = ax.get_yaxis_transform()
279
+ ax.plot(self.sound_extract, marker='o', markersize='1',
280
+ linewidth=1.5,alpha=0.3, color='blue' )
281
+ ax.plot(self.sound_extract_one_bit*np.max(np.abs(self.sound_extract)),
282
+ marker='o', markersize='1',
283
+ linewidth=1.5,alpha=0.3,color='red')
525
284
  xt = ax.get_xaxis_transform()
526
- _, silence_center_x = self._fit_triangular_signal_to_convoluted_env()
527
- # symbol_width_samples = 1e-3*SYMBOL_LENGTH
528
- # pulse_relative_pos = int(0.5*(0.5 - 1e-3*SYMBOL_LENGTH)*self.samplerate)
529
- approx_pulse_x = self._get_approx_pulse_position()
530
- ax.vlines(
531
- silence_center_x, 0.4, 0.6,
532
- transform=xt, linewidth=1, colors='black'
533
- )
534
- ax.vlines(
535
- approx_pulse_x, 0.1, 0.9,
536
- transform=xt, linewidth=1, colors='yellow'
537
- )
538
- bfsk_min = self._get_minimal_bfsk()
539
- ax.hlines(
540
- bfsk_min, 0, 1,
541
- transform=yt, linewidth=1, colors='red'
542
- )
543
- ax.hlines(
544
- trig_level, 0, 1,
545
- transform=yt, linewidth=1, colors='blue'
546
- )
547
- ax.hlines(
548
- -trig_level, 0, 1,
549
- transform=yt, linewidth=1, colors='blue'
550
- )
551
- ax.hlines(
552
- 0, 0, 1,
553
- transform=yt, linewidth=0.5, colors='black'
554
- )
555
- # plt.title(title)
285
+ yt = ax.get_yaxis_transform()
286
+ ax.hlines(self.pulse_detection_level, 0, 1,
287
+ transform=yt, alpha=0.3,
288
+ linewidth=2, colors='green')
556
289
  custom_lines = [
557
- Line2D([0], [0], color='black', lw=2),
558
290
  Line2D([0], [0], color='green', lw=2),
559
291
  Line2D([0], [0], color='blue', lw=2),
560
292
  Line2D([0], [0], color='red', lw=2),
561
- Line2D([0], [0], color='yellow', lw=2),
562
293
  ]
563
294
  ax.legend(
564
295
  custom_lines,
565
- 'sqr convolution,silence zone,pulse level,lower FSK ampl., approx puls.'.split(','),
296
+ 'detection level, signal, detected region'.split(','),
566
297
  loc='lower right')
567
- ax.plot(
568
- x_signal, signal,
569
- marker='o', markersize='1',
570
- linewidth=0.3, color='purple', alpha=0.3)
571
- ax.axvspan(
572
- start_silent_zone, end_silent_zone,
573
- alpha=0.1, color='green')
574
- ax.plot(
575
- x_convolution, scaled_convo,
576
- marker='.', markersize='0.2',
577
- linestyle='None', color='black', alpha=1)
578
- # linewidth=0.3, linestyle='None', color='black', alpha=0.3)
579
- # ax.set_xlabel('Decoder.sound_extract samples')
580
- if filename == None:
581
- plt.show()
582
- else:
583
- logger.debug('saving silence_analysis_plot to %s'%filename)
584
- plt.savefig(
585
- filename,
586
- format="png")
587
- plt.close()
588
-
589
- def _detect_sync_pulse_position(self):
590
- """
591
- Determines noise level during silence period and use it to detect the
592
- sync pulse position. Computes SN_ratio and stores it. Start searching
593
- around end of silent zone. Adjustment are made so a complete 0.5
594
- second signal is at the right of the starting search position so a
595
- complete 0.5 s word is available for decoding.
298
+ ax.set_title('Finding word and sync pulse')
299
+ plt.show()
596
300
 
597
- Returns the pulse position relative to the extract beginning.
598
-
599
- Sets self.detected_pulse_position to the returned value.
301
+ def get_time_in_sound_extract(self):
302
+ """
303
+ Tries to decode time present in self.sound_extract, if successfull
304
+ return a time dict, eg:{'version': 0, 'seconds':
305
+ 44, 'minutes': 57, 'hours': 19,
306
+ 'day': 1, 'month': 3, 'year offset': 1,
307
+ 'pulse at': 670451.2217 } otherwise return None
600
308
  """
601
- pulse_detection_level = self._get_pulse_detection_level()
602
- abs_signal = abs(self.sound_extract)
603
- mean_during_word = 2*abs_signal.mean()
604
- self.SN_ratio = 20*math.log10(mean_during_word/pulse_detection_level)
605
- logger.debug('SN ratio: %f dB'%(self.SN_ratio))
606
- search_pulse_start_point = self._get_approx_pulse_position()
607
- search_pulse_start_point -= 3*SYMBOL_LENGTH*1e-3*self.samplerate
608
- search_pulse_start_point = int(search_pulse_start_point)
609
- # search_pulse_start_point = end_silent_zone
610
- logger.debug('_get_pulse_position %i as starting point'%search_pulse_start_point)
611
- distance_to_end = len(self.sound_extract) - search_pulse_start_point
612
- time_to_end = distance_to_end/self.samplerate
613
- logger.debug('time_to_end after search_pulse_start_point %f sec'%
614
- time_to_end)
615
- if time_to_end < 0.5 + SYMBOL_LENGTH*1e-3: # sec
616
- logger.debug('search_pulse_start_point too near the end')
617
- # go back one second
618
- search_pulse_start_point -= self.samplerate
619
- logger.debug('new starting point %i'%search_pulse_start_point)
309
+ pulse_detected = self._detect_sync_pulse_position()
310
+ if not pulse_detected:
311
+ return None
312
+ symbols_data = self._get_symbols_data()
313
+ frequencies = [self._get_main_frequency(data_slice)
314
+ for data_slice in symbols_data ]
315
+ logger.debug('found frequencies %s'%frequencies)
316
+ def _get_bit_from_freq(freq):
317
+ mid_FSK = 0.5*(F1 + F2)
318
+ return '1' if freq > mid_FSK else '0'
319
+ bits = [_get_bit_from_freq(f) for f in frequencies]
320
+ bits_string = ''.join(bits)
321
+ logger.debug('giving bits: LSB %s MSB'%bits_string)
322
+
323
+ def _values_from_bits(bits):
324
+ word_payload_bits_positions = {
325
+ # start, finish (excluded)
326
+ 'version':(0,3), # 3 bits
327
+ 'seconds':(3,9), # 6 bits
328
+ 'minutes':(9,15),
329
+ 'hours':(15,20),
330
+ 'day':(20,25),
331
+ 'month':(25,29),
332
+ 'year offset':(29,34),
333
+ }
334
+ binary_words = { key : bits[slice(*value)]
335
+ for key, value
336
+ in word_payload_bits_positions.items()
337
+ }
338
+ int_values = { key : int(''.join(reversed(val)),2)
339
+ for key, val in binary_words.items()
340
+ }
341
+ return int_values
342
+ time_values = _values_from_bits(bits_string)
343
+ logger.debug(' decoded time %s'%time_values)
344
+ sync_pos_in_file = self.detected_pulse_position + \
345
+ self.sound_extract_position
346
+ time_values['pulse at'] = sync_pos_in_file
347
+ return time_values
348
+
349
+ def _detect_sync_pulse_position(self):
350
+ # sets self.detected_pulse_position, relative to sound_extract
351
+ #
352
+ regions = self.words_props # contains the sync pulse too
353
+ # len(self.words_props) should be 1 or 2 for vallid TTC
354
+ logger.debug('len() of words_props: %i'%len(self.words_props))
355
+ whole_region = [p for p in regions if 0.499 < _width(p)/self.samplerate < 0.512]
356
+ logger.debug('region widths %s'%[_width(p)/self.samplerate for p in regions])
357
+ logger.debug('number of whole_region %i'%len(whole_region))
358
+ if len(regions) == 1 and len(whole_region) != 1:
359
+ # oops
360
+ logger.debug('len(regions) == 1 and len(whole_region) != 1, failed')
361
+ return False #######################################################
362
+ if len(whole_region) > 1:
363
+ print('error in _detect_sync_pulse_position: len(whole_region) > 1 ')
364
+ return False #######################################################
365
+ if len(whole_region) == 1:
366
+ # sync pulse at the begining of this one
367
+ _, spike, _, _ = whole_region[0].bbox
620
368
  else:
621
- logger.debug('ok, ample room for 0.5 s word')
622
- search_pulse_start_point = max(search_pulse_start_point, 0)
623
- logger.debug('search_pulse_start_point: %i in extract'%
624
- search_pulse_start_point)
625
- abs_signal_after_silence = abs_signal[search_pulse_start_point:]
626
- # here the real searching with numpy.argmax()
627
- first_point = \
628
- np.argmax(abs_signal_after_silence > pulse_detection_level)
629
- first_point += search_pulse_start_point
630
- logger.debug('found sync pulse at %i in extract'%first_point)
631
- logger.debug('found sync pulse at %i in file'%(first_point
632
- + self.sound_extract_position))
633
- distance_to_end = len(self.sound_extract) - first_point
634
- time_to_end = distance_to_end/self.samplerate
635
- logger.debug('time_to_end after detected pulse %f sec'%
636
- time_to_end)
637
- if time_to_end < 0.5 + SYMBOL_LENGTH*1e-3: # sec
638
- logger.debug('detected_pulse_position too near the end')
639
- logger.debug('back one second')
640
- first_point -= self.samplerate
641
- logger.debug('new sync pulse at %i in extract'%first_point)
642
- logger.debug('new sync pulse at %i in file'%first_point)
369
+ # whole_region is [] (all fractionnal) and
370
+ # sync pulse at the begining of the 2nd region
371
+ _, spike, _, _ = regions[1].bbox
372
+ # but check there is still enough place for ten bits:
373
+ # 6 for secs + 3 for revision + blanck after sync
374
+ minimum_samples = int(self.samplerate*10*SYMBOL_LENGTH*1e-3)
375
+ whats_left = len(self.sound_extract) - spike
376
+ if whats_left < minimum_samples:
377
+ spike -= self.samplerate
378
+ # else: stay there, will decode seconds in whats_left
379
+ half_symbol_width = int(0.5*1e-3*SYMBOL_LENGTH*self.samplerate) # samples
380
+ left, right = (spike - half_symbol_width, spike+half_symbol_width)
381
+ spike_data = self.sound_extract[left:right]
382
+ biggest_positive = np.max(spike_data)
383
+ biggest_negative = np.min(spike_data)
384
+ if abs(biggest_negative) > biggest_positive:
385
+ # flip
386
+ spike_data = -1 * spike_data
387
+ def fit_line_until_negative():
388
+ import numpy as np
389
+ start = np.argmax(spike_data)
390
+ xs = [start]
391
+ ys = [spike_data[start]]
392
+ i = 1
393
+ while spike_data[start - i] > 0 and start - i >= 0:
394
+ xs.append(start - i)
395
+ ys.append(spike_data[start - i])
396
+ i += 1
397
+ # ax.scatter(xs, ys)
398
+ import numpy as np
399
+ coeff = np.polyfit(xs, ys, 1)
400
+ m, b = coeff
401
+ zero = int(-b/m)
402
+ # check if data is from USB audio and tweak
403
+ y_fit = np.poly1d(coeff)(xs)
404
+ err = abs(np.sum(np.abs(y_fit-ys))/np.mean(ys))
405
+ logger.debug('fit error for line in ramp: %f'%err)
406
+ if err < 0.01: #good fit so not analog
407
+ zero += 1
408
+ return zero
409
+ sync_sample = fit_line_until_negative() + left
410
+ logger.debug('sync pulse found at %i in extract, %i in file'%(
411
+ sync_sample, sync_sample + self.sound_extract_position))
412
+ self.detected_pulse_position = sync_sample
413
+ return True
414
+
415
+ def _get_symbols_data(self):
416
+ # part of extract AFTER sync pulse
417
+ whats_left = len(self.sound_extract) - self.detected_pulse_position # in samples
418
+ whats_left /= self.samplerate # in sec
419
+ whole_word_is_in_extr = whats_left > 0.512
420
+ if whole_word_is_in_extr:
421
+ # one region
422
+ logger.debug('word is in one sole region')
423
+ length_needed = round(0.5*self.samplerate)
424
+ length_needed += round(self.samplerate*SYMBOL_LENGTH*1e-3)
425
+ whole_word = self.sound_extract[self.detected_pulse_position:
426
+ self.detected_pulse_position + length_needed]
643
427
  else:
644
- logger.debug('ok, ample room for 0.5 s word after detected pulse')
645
- self.detected_pulse_position = first_point
646
- return first_point
647
-
648
- def _get_word_width_parameters(self):
649
- """
650
- Returns the parameters used to find and demodulate 2FSK word:
651
- presumed_symbol_length
652
- word_width_threshold
653
- search_end_position
654
- """
655
- abs_signal = abs(self.sound_extract)
656
- pulse_position = self._get_pulse_position()
657
- bfsk_min = self._get_minimal_bfsk()
658
- params = {'word_width_threshold': 0.8*bfsk_min}
659
- sr = self.samplerate
660
- presumed_symbol_length = SYMBOL_LENGTH*1e-3*sr
661
- # in ms, now in samples
662
- presumed_word_width = presumed_symbol_length * N_SYMBOLS_SAMD21
663
- # word includes sync pulse
664
- params['search_end_position'] = \
665
- int(presumed_symbol_length + pulse_position + \
666
- presumed_word_width) + 1700 # samples for headroom
667
- params['presumed_symbol_length'] = presumed_symbol_length
668
- return params
669
-
670
- def _get_BFSK_word_boundaries(self):
671
- n_bits = N_SYMBOLS_SAMD21 - 1
672
- sr = self.samplerate
673
- wwp = self._get_word_width_parameters()
674
- pulse_position = self._get_pulse_position()
675
- # search_start_position = wwp['search_start_position']
676
- search_end_position = wwp['search_end_position']
677
- word_width_threshold = wwp['word_width_threshold']
678
- word_extract = self.sound_extract[pulse_position :
679
- search_end_position]
680
- logger.debug('word left boundary search position: %i relative to extract'%
681
- (search_end_position))
682
- logger.debug('extract starting at %i in file (Decoder.sound_extract_position)'%
683
- self.sound_extract_position)
684
- flipped_extract = np.flip(np.abs(word_extract))
685
- right_boundary = len(word_extract) - \
686
- np.argmax(flipped_extract > word_width_threshold) + \
687
- pulse_position
688
- # left_boundary = np.argmax(
689
- # np.abs(word_extract) > word_width_threshold) + \
690
- # search_start_position
691
- left_boundary = pulse_position
692
- left_in_file = left_boundary + self.sound_extract_position
693
- right_in_file = right_boundary + self.sound_extract_position
694
- logger.debug('word boundaries: %i and %i in file'%
695
- (left_in_file, right_in_file))
696
- eff_symbol_length = 1e3*(right_boundary-left_boundary)/(n_bits*sr)
697
- logger.debug('effective symbol length %.4f ms '%
698
- eff_symbol_length)
699
- logger.debug('presumed symbol length %.4f ms '%SYMBOL_LENGTH)
700
- relative_error = (eff_symbol_length-SYMBOL_LENGTH)/SYMBOL_LENGTH
701
- status = True
702
- if relative_error > SYMBOL_LENGTH_TOLERANCE:
703
- logger.error(
704
- 'actual symbol length differs too much: %.2f vs %.2f ms'%
705
- (eff_symbol_length, SYMBOL_LENGTH))
706
- # return None, None
707
- status = False
708
- logger.debug(' relative discrepancy %.4f%%'%(abs(100*relative_error)))
709
- return status, left_boundary, right_boundary
710
-
711
- def _get_BFSK_symbols_boundaries(self):
712
- # returns indices of start of each slice and boundaries
713
- pulse_position = self._get_pulse_position()
714
- boundaries_OK, left_boundary, right_boundary = \
715
- self._get_BFSK_word_boundaries()
716
- if left_boundary is None:
717
- return None, None, None ############################################
718
- symbol_width_samples = \
719
- float(right_boundary - left_boundary)/N_SYMBOLS_SAMD21
720
- symbol_positions = symbol_width_samples * \
721
- np.arange(float(0), float(N_SYMBOLS_SAMD21 + 1)) + \
722
- pulse_position
723
- int_symb_positions = symbol_positions.round().astype(int)
724
- logger.debug('%i symbol positions %s samples in file'%
725
- (
726
- len(int_symb_positions),
727
- int_symb_positions + self.sound_extract_position)
728
- )
729
- return int_symb_positions[:-1], left_boundary, right_boundary
730
-
731
- def _values_from_bits(self, bits):
732
- word_payload_bits_positions = {
733
- 'version':(0,2),
734
- 'clock source':(2,3),
735
- 'seconds':(3,9),
736
- 'minutes':(9,15),
737
- 'hours':(15,20),
738
- 'day':(20,25),
739
- 'month':(25,29),
740
- 'year offset':(29,34),
741
- }
742
- binary_words = { key : bits[slice(*value)]
743
- for key, value
744
- in word_payload_bits_positions.items()
745
- }
746
- int_values = { key : self._get_int_from_binary_str(val)
747
- for key, val in binary_words.items()
748
- }
749
- logger.debug(' Demodulated values %s'%int_values)
750
- return int_values
751
-
752
- def _slice_sound_extract(self, symbols_indices):
753
- indices_left_shifted = deque(list(symbols_indices))
754
- indices_left_shifted.rotate(-1)
755
- all_intervals = list(zip(
756
- symbols_indices,
757
- indices_left_shifted
758
- ))
759
- word_intervals = all_intervals[1:-1]
760
- # [0, 11, 23, 31, 45] => [(11, 23), (23, 31), (31, 45)]
761
- logger.debug('slicing intervals, word_intervals = %s'%
762
- word_intervals)
763
- # skip sample after pulse, start at BFSK word
764
- filtered_sound_extract = self._band_pass_filter(self.sound_extract)
765
- slices = [filtered_sound_extract[slice(*pair)]
766
- for pair in word_intervals]
767
- np.set_printoptions(threshold=5)
768
- # logger.debug('data slices: \n%s'%pprint.pformat(slices))
769
- # raise Exception
428
+ # Two regions.
429
+ logger.debug('word is in two regions, will wrap past seconds')
430
+ # Consistency check: if not whole_word_is_in_extr
431
+ # check has been done so seconds are encoded in what s left
432
+ minimum_samples = round(self.samplerate*10*SYMBOL_LENGTH*1e-3)
433
+ if whats_left*self.samplerate < minimum_samples:
434
+ print('bug in _get_data_symbol():')
435
+ print(' whats_left*self.samplerate < minimum_samples')
436
+ # Should now build a whole 0.5 sec word by joining remaining data
437
+ # from previous second beep
438
+ left_piece = self.sound_extract[self.detected_pulse_position:]
439
+ one_second_before_idx = round(len(self.sound_extract) - self.samplerate)
440
+ length_needed = round(0.5*self.samplerate - len(left_piece))
441
+ length_needed += round(self.samplerate*SYMBOL_LENGTH*1e-3)
442
+ right_piece = self.sound_extract[one_second_before_idx:
443
+ one_second_before_idx + length_needed]
444
+ whole_word = np.concatenate((left_piece, right_piece))
445
+ logger.debug('two chunks lengths: %i %i samples'%(len(left_piece),
446
+ len(right_piece)))
447
+ # search for word start (some jitter because of Teensy Audio Lib)
448
+ symbol_length = round(self.samplerate*SYMBOL_LENGTH*1e-3)
449
+ start = round(0.5*symbol_length) # half symbol
450
+ end = start + symbol_length
451
+ word_begining = whole_word[start:]
452
+ # word_one_bit = np.abs(word_begining)>self.pulse_detection_level
453
+ # N_ones = round(1.5*SYMBOL_LENGTH*1e-3*self.samplerate) # so it includes sync pulse
454
+ # word_one_bit = closing(word_one_bit, np.ones(N_ones))
455
+ gt_detection_level = np.argwhere(np.abs(word_begining)>self.pulse_detection_level)
456
+ # print(gt_detection_level)
457
+ # plt.plot(word_one_bit)
458
+ # plt.plot(word_begining/abs(np.max(word_begining)))
459
+ # plt.show()
460
+ word_start = gt_detection_level[0][0]
461
+ word_end = gt_detection_level[-1][0]
462
+ self.effective_word_duration = (word_end - word_start)/self.samplerate
463
+ logger.debug('effective_word_duration %f s'%self.effective_word_duration)
464
+ uCTRLR_error = self.effective_word_duration/((N_SYMBOLS -1)*SYMBOL_LENGTH*1e-3)
465
+ logger.debug('uCTRLR_error %f (time ratio)'%uCTRLR_error)
466
+ word_start += start # relative to Decoder extract
467
+ # check if gap is indeed less than TEENSY_MAX_LAG
468
+ silence_length = word_start
469
+ gap = silence_length - symbol_length
470
+ relative_gap = gap/(TEENSY_MAX_LAG*self.samplerate)
471
+ logger.debug('Audio update() gap between sync pulse and word start: ')
472
+ logger.debug('%.2f ms (max value %.2f)'%(1e3*gap/self.samplerate,
473
+ 1e3*TEENSY_MAX_LAG))
474
+ logger.debug('relative audio_block gap %.2f'%(relative_gap))
475
+ if relative_gap > 1:
476
+ print('bug with relative_gap')
477
+ symbol_width_samples_theor = self.samplerate*SYMBOL_LENGTH*1e-3
478
+ symbol_width_samples_eff = self.effective_word_duration * \
479
+ self.samplerate/(N_SYMBOLS - 1)
480
+ logger.debug('symbol width %i theo; %i effective (samples)'%(
481
+ symbol_width_samples_theor,
482
+ symbol_width_samples_eff))
483
+ symbol_positions = symbol_width_samples_eff * \
484
+ np.arange(float(0), float(N_SYMBOLS - 1)) + word_start
485
+ # symbols_indices contains 34 start of symbols (samples)
486
+ symbols_indices = symbol_positions.round().astype(int)
487
+ if self.do_plots:
488
+ fig, ax = plt.subplots()
489
+ ax.plot(whole_word, marker='o', markersize='1',
490
+ linewidth=1.5,alpha=0.3, color='blue' )
491
+ xt = ax.get_xaxis_transform()
492
+ for x in symbols_indices:
493
+ ax.vlines(x, 0, 1,
494
+ transform=xt,
495
+ linewidth=0.6, colors='green')
496
+ ax.set_title('Slicing the 34 bits word:')
497
+ plt.show()
498
+ slice_width = round(SYMBOL_LENGTH*1e-3*self.samplerate)
499
+ slices = [whole_word[i:i+slice_width] for i in symbols_indices]
770
500
  return slices
771
501
 
772
502
  def _get_main_frequency(self, symbol_data):
@@ -777,195 +507,6 @@ class Decoder:
777
507
  freq_in_hertz = abs(freq * self.samplerate)
778
508
  return int(round(freq_in_hertz))
779
509
 
780
- # def _get_bit_from_freq(self, freq):
781
- # if math.isclose(freq, F1, abs_tol=FSK_TOLERANCE):
782
- # return '0'
783
- # if math.isclose(freq, F2, abs_tol=FSK_TOLERANCE):
784
- # return '1'
785
- # else:
786
- # return None
787
-
788
- def _get_bit_from_freq(self, freq):
789
- mid_FSK = 0.5*(F1 + F2)
790
- return '1' if freq > mid_FSK else '0'
791
-
792
- def _get_int_from_binary_str(self, string_of_01s):
793
- return int(''.join(reversed(string_of_01s)),2)
794
- # LSB is leftmost in TicTacCode
795
-
796
- def _demod_values_are_OK(self, values_dict):
797
- # TODO: use _get_timedate_from_dict rather (catching any ValueError)
798
- ranges = {
799
- 'seconds': range(60),
800
- 'minutes': range(60),
801
- 'hours': range(24),
802
- 'day': range(1,32), # 32 ?
803
- 'month': range(1,13),
804
- }
805
- for key in ranges:
806
- val = values_dict[key]
807
- ok = val in ranges[key]
808
- logger.debug(
809
- 'checking range for %s: %i, Ok? %s'%(key, val, ok))
810
- if not ok:
811
- logger.error('demodulated value is out of range')
812
- return False
813
- return True
814
-
815
- def _plot_slices(self, sync_pulse, symbols_indices, word_lft, word_rght,
816
- title=None, filename=None):
817
- # save figure in filename if set, otherwise
818
- # start an interactive plot, title is for matplotlib
819
- signal = self.sound_extract
820
- # signal = self._band_pass_filter(signal)
821
- start = self.sound_extract_position
822
- x_signal_in_file = range(
823
- start,
824
- start + len(signal)
825
- )
826
- wwp = self._get_word_width_parameters()
827
- start_silent_zone, end_silent_zone = self._get_silent_zone_indices()
828
- search_end_position = wwp['search_end_position'] + start
829
- logger.debug('doing slice plot')
830
- fig, ax = plt.subplots()
831
- plt.title(title)
832
- ax.plot(
833
- x_signal_in_file, signal,
834
- marker='.', markersize='1',
835
- linewidth=0.3, color='purple', alpha=0.3)
836
- yt = ax.get_yaxis_transform()
837
- ax.hlines(
838
- wwp['word_width_threshold'], 0, 1,
839
- transform=yt, linewidth=0.6, colors='green')
840
- ax.hlines(
841
- 0, 0, 1,
842
- transform=yt, linewidth=0.6, colors='black')
843
- ax.hlines(
844
- -wwp['word_width_threshold'], 0, 1,
845
- transform=yt, linewidth=0.6, colors='green')
846
- pulse_level = self._get_pulse_detection_level()
847
- ax.hlines(
848
- pulse_level, 0, 1,
849
- transform=yt, linewidth=0.6, colors='blue')
850
- ax.hlines(
851
- -pulse_level, 0, 1,
852
- transform=yt, linewidth=0.6, colors='blue')
853
- xt = ax.get_xaxis_transform()
854
- # ax.vlines(
855
- # search_start_position,
856
- # 0, 1, transform=xt, linewidth=0.6, colors='blue')
857
- ax.vlines(
858
- search_end_position,
859
- 0, 1, transform=xt, linewidth=0.6, colors='blue')
860
- ax.plot(
861
- [sync_pulse + start], [0],
862
- marker='D', markersize='7',
863
- linewidth=0.3, color='blue', alpha=0.3)
864
- ax.plot(
865
- [start_silent_zone + start], [0],
866
- marker='>', markersize='10',
867
- linewidth=0.3, color='green', alpha=0.3)
868
- ax.plot(
869
- [end_silent_zone + start], [0],
870
- marker='<', markersize='10',
871
- linewidth=0.3, color='green', alpha=0.3)
872
- boundaries_OK, word_lft, word_rght = \
873
- self._get_BFSK_word_boundaries()
874
- ax.vlines(
875
- word_lft + start, 0, 1,
876
- transform=ax.get_xaxis_transform(),
877
- linewidth=0.6, colors='red')
878
- ax.vlines(
879
- word_rght + start, 0, 1,
880
- transform=ax.get_xaxis_transform(),
881
- linewidth=0.6, colors='red')
882
- for x in symbols_indices + self.sound_extract_position:
883
- ax.vlines(
884
- x, 0, 1,
885
- transform=ax.get_xaxis_transform(),
886
- linewidth=0.3, colors='green')
887
- ax.set_xlabel(
888
- 'samples in file')
889
- plt.xlim(
890
- [sync_pulse - 300 + start, wwp['search_end_position'] + 400 + start])
891
- if filename == None:
892
- plt.show()
893
- else:
894
- plt.ylim(
895
- [-1.5*wwp['word_width_threshold'],
896
- 1.1*signal.max()])
897
- height = 1000
898
- plt.savefig(
899
- filename,
900
- format='png',
901
- dpi=height/fig.get_size_inches()[1])
902
- plt.close()
903
- logger.debug('done slice plot')
904
-
905
- def _band_pass_filter(self, data):
906
- # return filtered data
907
- def _bandpass(data: np.ndarray, edges: list[float], sample_rate: float, poles: int = 5):
908
- sos = scipy.signal.butter(poles, edges, 'bandpass', fs=sample_rate, output='sos')
909
- filtered_data = scipy.signal.sosfiltfilt(sos, data)
910
- return filtered_data
911
- sample_rate = self.samplerate
912
- times = np.arange(len(data))/sample_rate
913
- return _bandpass(data, [BPF_LOW_FRQ, BPF_HIGH_FRQ], sample_rate)
914
-
915
- def get_time_in_sound_extract(self, plots):
916
- if self.sound_extract is None:
917
- return None ########################################################
918
- if plots:
919
- self.make_silence_analysis_plot()
920
- pulse_position = self._detect_sync_pulse_position()
921
- pulse_pos_in_file = pulse_position + self.sound_extract_position
922
- pulse_position_sec = pulse_pos_in_file/self.samplerate
923
- logger.debug('found sync pulse at sample %i in file'%pulse_pos_in_file)
924
- symbols_indices, word_lft, word_rght = \
925
- self._get_BFSK_symbols_boundaries()
926
- if plots:
927
- title = 'Bit slicing at %s, %.2f s'%(pulse_pos_in_file,
928
- pulse_position_sec)
929
- # self.make_silence_analysis_plot()
930
- logger.debug('calling _plot_slices()')
931
- self._plot_slices(pulse_position, symbols_indices, word_lft,
932
- word_rght, title)
933
- if symbols_indices is None:
934
- return None ########################################################
935
- sliced_data = self._slice_sound_extract(symbols_indices)
936
- frequencies = [self._get_main_frequency(data_slice)
937
- for data_slice
938
- in sliced_data
939
- ]
940
- logger.debug('frequencies = %s'%frequencies)
941
- sr = self.samplerate
942
- n_bits = N_SYMBOLS_SAMD21 - 1
943
- eff_symbol_length = 1e3*(word_rght-word_lft)/(n_bits*sr)
944
- length_ratio = eff_symbol_length / SYMBOL_LENGTH
945
- logger.debug('symbol length_ratio (eff/supposed) %f'%length_ratio)
946
- corrected_freq = np.array(frequencies)*length_ratio
947
- logger.debug('corrected freq (using symbol length) = %s'%corrected_freq)
948
- bits = [self._get_bit_from_freq(f) for f in corrected_freq]
949
- for i, bit in enumerate(bits):
950
- if bit == None:
951
- logger.warning('cant decode frequency %i for bit at %i-%i'%(
952
- corrected_freq[i],
953
- symbols_indices[i],
954
- symbols_indices[i+1]))
955
- if None in bits:
956
- return None ########################################################
957
- bits_string = ''.join(bits)
958
- logger.debug('bits = %s'%bits_string)
959
- time_values = self._values_from_bits(bits_string)
960
- time_values['pulse at'] = (pulse_position +
961
- self.sound_extract_position -
962
- SAMD21_LATENCY*1e-6*self.samplerate)
963
- time_values['clock source'] = 'GPS' \
964
- if time_values['clock source'] == 1 else 'RTC'
965
- if self._demod_values_are_OK(time_values):
966
- return time_values
967
- else:
968
- return None
969
510
 
970
511
  class Recording:
971
512
  """
@@ -1047,7 +588,7 @@ class Recording:
1047
588
 
1048
589
  """
1049
590
 
1050
- def __init__(self, media):
591
+ def __init__(self, media, do_plots=False):
1051
592
  """
1052
593
  If multifile recording, AVfilename is sox merged audio file;
1053
594
  Set AVfilename string and check if file exists, does not read
@@ -1097,6 +638,7 @@ class Recording:
1097
638
  self.final_synced_file = None
1098
639
  self.synced_audio = None
1099
640
  self.new_rec_name = media.path.name
641
+ self.do_plots = do_plots
1100
642
  logger.debug('__init__ Recording object %s'%self.__repr__())
1101
643
  logger.debug(' in directory %s'%self.AVpath.parent)
1102
644
  recording_init_fail = ''
@@ -1119,7 +661,7 @@ class Recording:
1119
661
  elif self.get_duration() < MINIMUM_LENGTH:
1120
662
  recording_init_fail = 'file too short, %f s\n'%self.get_duration()
1121
663
  if recording_init_fail == '': # success
1122
- self.decoder = Decoder(self)
664
+ self.decoder = Decoder(self, do_plots)
1123
665
  # self._set_multi_files_siblings()
1124
666
  self._check_for_camera_error_correction()
1125
667
  else:
@@ -1171,7 +713,7 @@ class Recording:
1171
713
  """
1172
714
  if self.valid_sound:
1173
715
  val = sox.file_info.duration(_pathname(self.valid_sound))
1174
- logger.debug('duration of valid_sound %f'%val)
716
+ logger.debug('sox duration of valid_sound %f for %s'%(val,_pathname(self.valid_sound)))
1175
717
  return val #########################################################
1176
718
  else:
1177
719
  if self.probe is None:
@@ -1181,7 +723,7 @@ class Recording:
1181
723
  except:
1182
724
  logger.error('oups, cant find duration from ffprobe')
1183
725
  raise Exception('stopping here')
1184
- logger.debug('ffprobed duration is: %f sec'%probed_duration)
726
+ logger.debug('ffprobed duration is: %f sec for %s'%(probed_duration, self))
1185
727
  return probed_duration # duration in s
1186
728
 
1187
729
  def get_original_duration(self):
@@ -1238,16 +780,16 @@ class Recording:
1238
780
  end = self.get_end_time()
1239
781
  return start < datetime and datetime < end
1240
782
 
1241
- def _find_time_around(self, time, plots):
783
+ def _find_time_around(self, time):
1242
784
  """
1243
785
  Actually reads sound data and tries to decode it
1244
786
  through decoder object, if successful return a time dict, eg:
1245
- {'version': 0, 'clock source': 'GPS', 'seconds': 44, 'minutes': 57,
787
+ {'version': 0, 'seconds': 44, 'minutes': 57,
1246
788
  'hours': 19, 'day': 1, 'month': 3, 'year offset': 1,
1247
789
  'pulse at': 670451.2217 }
1248
790
  otherwise return None
1249
791
  """
1250
- if time < 0:
792
+ if time < 0: # negative = referenced from the end
1251
793
  there = self.get_duration() + time
1252
794
  else:
1253
795
  there = time
@@ -1255,7 +797,7 @@ class Recording:
1255
797
  if self.TicTacCode_channel is None:
1256
798
  return None
1257
799
  else:
1258
- return self.decoder.get_time_in_sound_extract(plots)
800
+ return self.decoder.get_time_in_sound_extract()
1259
801
 
1260
802
  def _get_timedate_from_dict(self, time_dict):
1261
803
  try:
@@ -1291,8 +833,6 @@ class Recording:
1291
833
 
1292
834
  Returns
1293
835
  -------
1294
- TYPE
1295
- DESCRIPTION.
1296
836
 
1297
837
  """
1298
838
  if t1 == None or t2 == None:
@@ -1362,7 +902,7 @@ class Recording:
1362
902
  t2 = later_recording.get_start_time()
1363
903
  return t2 - t1
1364
904
 
1365
- def get_start_time(self, plots=False):
905
+ def get_start_time(self):
1366
906
  """
1367
907
  Try to decode a TicTacCode_channel at start AND finish;
1368
908
  if successful, returns a datetime.datetime instance;
@@ -1372,7 +912,7 @@ class Recording:
1372
912
  if self.start_time is not None:
1373
913
  return self.start_time #############################################
1374
914
  cached_times = {}
1375
- def find_time(t_sec, plots=False):
915
+ def find_time(t_sec):
1376
916
  time_k = int(t_sec)
1377
917
  # if cached_times.has_key(time_k):
1378
918
  if CACHING and time_k in cached_times:
@@ -1380,7 +920,7 @@ class Recording:
1380
920
  return cached_times[time_k] ####################################
1381
921
  else:
1382
922
  logger.debug('_find_time_around() for t=%s s not cached'%time_k)
1383
- new_t = self._find_time_around(t_sec, plots)
923
+ new_t = self._find_time_around(t_sec)
1384
924
  cached_times[time_k] = new_t
1385
925
  return new_t
1386
926
  for i, pair in enumerate(TRIAL_TIMES):
@@ -1393,12 +933,12 @@ class Recording:
1393
933
  logger.warning('More than one trial: #%i/%i'%(i+1,
1394
934
  len(TRIAL_TIMES)))
1395
935
  # time_around_beginning = self._find_time_around(near_beg)
1396
- time_around_beginning = find_time(near_beg, plots)
936
+ time_around_beginning = find_time(near_beg)
1397
937
  if self.TicTacCode_channel is None:
1398
938
  return None ####################################################
1399
939
  logger.debug('Trial #%i, end at %f'%(i+1, near_end))
1400
940
  # time_around_end = self._find_time_around(near_end)
1401
- time_around_end = find_time(near_end, plots)
941
+ time_around_end = find_time(near_end)
1402
942
  logger.debug('trial result, time_around_beginning:\n %s'%
1403
943
  (time_around_beginning))
1404
944
  logger.debug('trial result, time_around_end:\n %s'%
@@ -1436,133 +976,6 @@ class Recording:
1436
976
  self.valid_sound = self.AVpath
1437
977
  return start_UTC
1438
978
 
1439
- # def _strip_TTC_and_Null(self) -> tempfile.NamedTemporaryFile:
1440
- # """
1441
- # TTC is stripped from original audio and a tempfile.NamedTemporaryFile is
1442
- # returned. If the original audio is stereo, this is simply the audio
1443
- # without the TicTacCode channel, so this fct returns a mono wav
1444
- # tempfile. But if the original audio is multitrack, two possibilities:
1445
-
1446
- # A- there's a track.txt file declaring null channels (with '0' tags)
1447
- # then those tracks are excluded too (a check is done those tracks
1448
- # have low signal and warns the user otherwise)
1449
-
1450
- # B- it's a multitrack recording but without track declaration
1451
- # (no track.txt was found in the device folder): all the tracks are
1452
- # returned into a multiwav tempfile (except TTC).
1453
-
1454
- # Notes:
1455
- # 'track.txt' is defined in device_scanner.TRACKSFN
1456
-
1457
- # Beware of track indexing: sox starts indexing tracks at 1 but code
1458
- # here uses zero based indexing.
1459
- # """
1460
- # tracks_file = self.device.folder/device_scanner.TRACKSFN
1461
- # input_file = _pathname(self.AVpath)
1462
- # # n_channels = sox.file_info.channels(input_file) # eg 2
1463
- # n_channels = self.device.n_chan
1464
- # sox_TicTacCode_channel = self.TicTacCode_channel + 1 # sox start at 1
1465
- # if n_channels == 2:
1466
- # logger.debug('stereo, so only excluding TTC (ZB idx) %i'%
1467
- # self.TicTacCode_channel)
1468
- # return self._sox_strip(input_file, [self.TicTacCode_channel]) ######
1469
- # #
1470
- # # First a check is done if the ttc tracks concur: the track detected
1471
- # # by the Decoder class, stored in Recording.TicTacCode_channel VS the
1472
- # # track declared by the user, Tracks.ttc (see device_scanner.py). If
1473
- # # not, warn the user and exit.
1474
- # trax = self.device.tracks # a Tracks dataclass instance, if any
1475
- # logger.debug('trax %s'%trax)
1476
- # if trax == None:
1477
- # return self._sox_strip(input_file, [self.TicTacCode_channel]) ######
1478
- # else:
1479
- # logger.debug('ttc channel declared for the device: %i, ttc detected: %i, non zero base indexing'%
1480
- # (trax.ttc, sox_TicTacCode_channel))
1481
- # if trax.ttc != sox_TicTacCode_channel: # warn and quit
1482
- # print('Error: TicTacCode channel detected is [gold1]%i[/gold1]'%
1483
- # sox_TicTacCode_channel, end=' ')
1484
- # print('and the file [gold1]%s[/gold1] specifies channel [gold1]%i[/gold1],'%
1485
- # (tracks_file, trax.ttc))
1486
- # print('Please correct the discrepancy and rerun. Quitting.')
1487
- # sys.exit(1)
1488
- # track_is_declared_audio = [ tag not in ['ttc','0','tc']
1489
- # for tag in trax.rawtrx]
1490
- # logger.debug('from tracks_file %s'%tracks_file)
1491
- # logger.debug('track_is_declared_audio (not ttc or 0): %s'%
1492
- # track_is_declared_audio)
1493
- # # Now find out which files are silent, ie those with sox stats "RMS lev
1494
- # # db" value inferior to DB_RMS_SILENCE_SOX (typ -50 -60 dbFS) from the
1495
- # # sox "stat" command. Ex output belwow:
1496
- # #
1497
- # # sox input.wav -n channels stats -w 0.01
1498
- # #
1499
- # # Overall Ch1 Ch2 Ch3 Ch4
1500
- # # DC offset -0.000047 -0.000047 -0.000047 -0.000016 -0.000031
1501
- # # Min level -0.060913 -0.060913 -0.048523 -0.036438 -0.002777
1502
- # # Max level 0.050201 0.050201 0.048767 0.039032 0.002838
1503
- # # Pk lev dB -24.31 -24.31 -26.24 -28.17 -50.94
1504
- # # RMS lev dB -40.33 -55.29 -53.70 -34.41 -59.75 <- this line
1505
- # # RMS Pk dB -28.39 -28.39 -30.90 -31.20 -55.79
1506
- # # RMS Tr dB -97.42 -79.66 -75.87 -97.42 -96.09
1507
- # # Crest factor - 35.41 23.61 2.05 2.76
1508
- # # Flat factor 6.93 0.00 0.00 0.97 10.63
1509
- # # Pk count 10.2 2 2 17 20
1510
- # # Bit-depth 12/16 12/16 12/16 12/16 8/16
1511
- # # Num samples 11.2M
1512
- # # Length s 232.780
1513
- # # Scale max 1.000000
1514
- # # Window s 0.010
1515
- # args = ['sox', input_file, '-n', 'channels', 'stats', '-w', '0.01']
1516
- # _, _, stat_output = sox.core.sox(args)
1517
- # logger.debug('sox stat output: \n%s'%stat_output)
1518
- # sox_RMS_lev_dB = stat_output.split('\n')[5].split()[4:]
1519
- # logger.debug('Rec %s'%self)
1520
- # logger.debug('Sox RMS %s, n_channels: %i'%(sox_RMS_lev_dB, n_channels))
1521
- # # valid audio is non silent and not ttc
1522
- # track_is_active = [float(db) > DB_RMS_SILENCE_SOX
1523
- # if idx + 1 != sox_TicTacCode_channel else False
1524
- # for idx, db in enumerate(sox_RMS_lev_dB)]
1525
- # logger.debug('track_is_active %s'%track_is_active)
1526
- # # Stored in self.device.tracks and as declared by the user, a track is
1527
- # # either:
1528
- # #
1529
- # # - an active track (because a name was given to it)
1530
- # # - the ttc track (as identified by the user)
1531
- # # - a muted track (declared by a "0" in tracks.txt)
1532
- # #
1533
- # # the following checks active tracks are effectively non silent and
1534
- # # muted tracks are effectively silent (warn the user if not but
1535
- # # proceed, giving priority to status declared in the tracks.txt file.
1536
- # # eg a non silent track will be discarded if the user tagged it with
1537
- # # a "0")
1538
- # declared_and_detected_are_same = all([a==b for a,b
1539
- # in zip(track_is_declared_audio, track_is_active)])
1540
- # logger.debug('declared_and_detected_are_same: %s'%
1541
- # declared_and_detected_are_same)
1542
- # if not declared_and_detected_are_same:
1543
- # print('Warning, the file [gold1]%s[/gold1] specifies channel usage'%
1544
- # (tracks_file))
1545
- # print('and some muted tracks are not silent (or the inverse, see below),')
1546
- # print('will proceed but if it\'s an error do necessary corrections and rerun.\n')
1547
- # table = Table(title="Tracks status")
1548
- # table.add_column("track #", justify="center", style='gold1')
1549
- # table.add_column("RMS Level", justify="center", style='gold1')
1550
- # table.add_column("Declared", justify="center", style='gold1')
1551
- # for n in range(n_channels):
1552
- # table.add_row(str(n+1), '%.0f dBFS'%float(sox_RMS_lev_dB[n]),
1553
- # trax.rawtrx[n])
1554
- # console = Console()
1555
- # console.print(table)
1556
- # if trax:
1557
- # excluded_channels = [i for i in range(n_channels)
1558
- # if not track_is_declared_audio[i]]
1559
- # else:
1560
- # excluded_channels = [self.TicTacCode_channel]
1561
- # logger.debug('excluded_channels %s (ZB idx)'%excluded_channels)
1562
- # return self._sox_strip(input_file, excluded_channels)
1563
-
1564
-
1565
-
1566
979
  def _sox_strip(self, audio_file, excluded_channels) -> tempfile.NamedTemporaryFile:
1567
980
  # building dict according to pysox.remix format.
1568
981
  # https://pysox.readthedocs.io/en/latest/api.html#sox.transform.Transformer.remix
@@ -1649,6 +1062,7 @@ class Recording:
1649
1062
  def get_samplerate(self):
1650
1063
  # return int samplerate (nominal)
1651
1064
  string = self._ffprobe_audio_stream()['sample_rate']
1065
+ logger.debug('ffprobe samplerate: %s'%string)
1652
1066
  return eval(string) # eg eval(24000/1001)
1653
1067
 
1654
1068
  def get_framerate(self):
@@ -1717,13 +1131,16 @@ class Recording:
1717
1131
 
1718
1132
  def _read_sound_find_TicTacCode(self, time_where, chunk_length):
1719
1133
  """
1720
- Loads audio data reading from self.AVpath;
1721
- Split data into channels if stereo;
1722
- Send this data to Decoder object with set_sound_extract_and_sr() to find
1134
+ If this is called for the first time for the recording, it loads audio
1135
+ data reading from self.AVpath; Split data into channels if stereo; Send
1136
+ this data to Decoder object with set_sound_extract_and_sr() to find
1723
1137
  which channel contains a TicTacCode track and sets TicTacCode_channel
1724
- accordingly: a tuple (#file, #chan). On exit, self.decoder.sound_extract
1725
- contains TicTacCode data ready to be demodulated. If not, self.TicTacCode_channel
1726
- is set to None.
1138
+ accordingly (index of channel). On exit, self.decoder.sound_extract
1139
+ contains TicTacCode data ready to be demodulated. If not,
1140
+ self.TicTacCode_channel is set to None.
1141
+
1142
+ If this has been called before (checking self.TicTacCode_channel) then
1143
+ is simply read the audio in and calls set_sound_extract_and_sr().
1727
1144
 
1728
1145
  Args:
1729
1146
  time_where : float
@@ -1735,7 +1152,8 @@ class Recording:
1735
1152
  self.decoder.set_sound_extract_and_sr()
1736
1153
 
1737
1154
  Sets:
1738
- self.TicTacCode_channel
1155
+ self.TicTacCode_channel = index of TTC chan
1156
+ self.device.ttc = index of TTC chan
1739
1157
 
1740
1158
  Returns:
1741
1159
  this Recording instance
@@ -1751,14 +1169,15 @@ class Recording:
1751
1169
  return #############################################################
1752
1170
  logger.debug('will read around %.2f sec'%time_where)
1753
1171
  dryrun = (ffmpeg
1754
- .input(str(path), ss=time_where, t=chunk_length)
1172
+ .input(str(path))
1755
1173
  .output('pipe:', format='s16le', acodec='pcm_s16le')
1756
1174
  .get_args())
1757
1175
  dryrun = ' '.join(dryrun)
1758
1176
  logger.debug('using ffmpeg-python built args to pipe wav file into numpy array:\nffmpeg %s'%dryrun)
1759
1177
  try:
1760
1178
  out, _ = (ffmpeg
1761
- .input(str(path), ss=time_where, t=chunk_length)
1179
+ # .input(str(path), ss=time_where, t=chunk_length)
1180
+ .input(str(path))
1762
1181
  .output('pipe:', format='s16le', acodec='pcm_s16le')
1763
1182
  .global_args("-loglevel", "quiet")
1764
1183
  .global_args("-nostats")
@@ -1766,28 +1185,54 @@ class Recording:
1766
1185
  .run(capture_stdout=True))
1767
1186
  data = np.frombuffer(out, np.int16)
1768
1187
  except ffmpeg.Error as e:
1769
- print('error',e.stderr)
1188
+ print('error',e.stderr)
1189
+ sound_data_var = np.std(data)
1190
+ logger.debug('extracting sound, ffmpeg output:%s with variance %f'%(data,
1191
+ sound_data_var))
1770
1192
  sound_extract_position = int(self.get_samplerate()*time_where) # from sec to samples
1771
1193
  n_chan = self.get_audio_channels_nbr()
1772
1194
  if n_chan == 1 and not self.is_video():
1773
1195
  logger.warning('file is sound mono')
1196
+ if np.isclose(sound_data_var, 0, rtol=1e-2):
1197
+ logger.warning("ffmpeg can't extract audio from %s"%self.AVpath)
1198
+ # from 1D interleaved channels to [chan1, chan2, chanN]
1774
1199
  all_channels_data = data.reshape(int(len(data)/n_chan),n_chan).T
1775
- for i_chan, chan_dat in enumerate(all_channels_data):
1776
- logger.debug('testing chan %i'%i_chan)
1200
+ if self.TicTacCode_channel == None:
1201
+ logger.debug('first call, will loop through all %i channels'%len(
1202
+ all_channels_data))
1203
+ for i_chan, chan_dat in enumerate(all_channels_data):
1204
+ logger.debug('testing chan %i'%i_chan)
1205
+ start_idx = round(time_where*self.get_samplerate())
1206
+ extract_length = round(chunk_length*self.get_samplerate())
1207
+ end_idx = start_idx + extract_length
1208
+ extract_audio_data = chan_dat[start_idx:end_idx]
1209
+ decoder.set_sound_extract_and_sr(
1210
+ extract_audio_data,
1211
+ self.get_samplerate(),
1212
+ sound_extract_position
1213
+ )
1214
+ if decoder.extract_seems_TicTacCode():
1215
+ self.TicTacCode_channel = i_chan
1216
+ self.device.ttc = i_chan
1217
+ logger.debug('found TicTacCode channel: chan #%i'%
1218
+ self.TicTacCode_channel)
1219
+ return self ################################################
1220
+ # end of loop: none found
1221
+ # self.TicTacCode_channel = None # was None already
1222
+ logger.warning('found no TicTacCode channel')
1223
+ else:
1224
+ logger.debug('been here before, TTC chan is %i'%
1225
+ self.TicTacCode_channel)
1226
+ start_idx = round(time_where*self.get_samplerate())
1227
+ extract_length = round(chunk_length*self.get_samplerate())
1228
+ end_idx = start_idx + extract_length
1229
+ chan_dat = all_channels_data[self.TicTacCode_channel]
1230
+ extract_audio_data = chan_dat[start_idx:end_idx]
1777
1231
  decoder.set_sound_extract_and_sr(
1778
- chan_dat,
1232
+ extract_audio_data,
1779
1233
  self.get_samplerate(),
1780
1234
  sound_extract_position
1781
1235
  )
1782
- if decoder.extract_seems_TicTacCode():
1783
- self.TicTacCode_channel = i_chan
1784
- self.device.ttc = i_chan
1785
- logger.debug('find TicTacCode channel, chan #%i'%
1786
- self.TicTacCode_channel)
1787
- return self ####################################################
1788
- # end of loop: none found
1789
- self.TicTacCode_channel = None
1790
- logger.warning('found no TicTacCode channel')
1791
1236
  return self
1792
1237
 
1793
1238
  def seems_to_have_TicTacCode_at_beginning(self):