tictacsync 0.5a0__py3-none-any.whl → 0.6a0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of tictacsync might be problematic. Click here for more details.

tictacsync/yaltc.py CHANGED
@@ -5,7 +5,7 @@ import lmfit, tempfile
5
5
  # from skimage.morphology import closing, square
6
6
  from numpy import arcsin, sin, pi
7
7
  from matplotlib.lines import Line2D
8
- import math, re, os, sys
8
+ import math, re, os, sys, itertools
9
9
  import sox
10
10
  from subprocess import Popen, PIPE
11
11
  from pathlib import Path
@@ -18,6 +18,8 @@ from datetime import datetime, timezone, timedelta
18
18
  from collections import deque
19
19
  from loguru import logger
20
20
  from sklearn.mixture import GaussianMixture
21
+ from skimage.morphology import closing, erosion, remove_small_objects
22
+ from skimage.measure import regionprops, label
21
23
  import ffmpeg, shutil
22
24
  from rich import print
23
25
  from rich.console import Console
@@ -28,6 +30,8 @@ try:
28
30
  except:
29
31
  import device_scanner
30
32
 
33
+ TEENSY_MAX_LAG = 128/44100 # sec, duration of a default length audio block
34
+
31
35
 
32
36
  CACHING = True
33
37
  DEL_TEMP = False
@@ -41,6 +45,13 @@ WORDWIDTHFACTOR = 2
41
45
 
42
46
  OVER_NOISE_SYNC_DETECT_LEVEL = 2
43
47
 
48
+ ################## pasted from FSKfreqCalculator.py output:
49
+ F1 = 630.00 # Hertz
50
+ F2 = 1190.00 # Hz , both from FSKfreqCalculator output
51
+ SYMBOL_LENGTH = 14.286 # ms, from FSKfreqCalculator.py
52
+ N_SYMBOLS = 35 # including sync pulse
53
+ ##################
54
+
44
55
  MINIMUM_LENGTH = 4 # sec
45
56
  TRIAL_TIMES = [ # in seconds
46
57
  (0.5, -2),
@@ -52,18 +63,12 @@ TRIAL_TIMES = [ # in seconds
52
63
  (3.5, -2),
53
64
  (3.5, -3.5),
54
65
  ]
55
- SOUND_EXTRACT_LENGTH = 1.56 # second
66
+ SOUND_EXTRACT_LENGTH = (10*SYMBOL_LENGTH*1e-3 + 1) # second
56
67
  SYMBOL_LENGTH_TOLERANCE = 0.07 # relative
57
68
  FSK_TOLERANCE = 60 # Hz
58
69
  SAMD21_LATENCY = 63 # microseconds, for DAC conversion
59
70
  YEAR_ZERO = 2021
60
71
 
61
- ################## pasted from FSKfreqCalculator.py output:
62
- F1 = 630.00 # Hertz
63
- F2 = 1190.00 # Hz , both from FSKfreqCalculator output
64
- SYMBOL_LENGTH = 14.286 # ms, from FSKfreqCalculator.py
65
- N_SYMBOLS_SAMD21 = 35 # including sync pulse
66
- ##################
67
72
 
68
73
  BPF_LOW_FRQ, BPF_HIGH_FRQ = (0.5*F1, 2*F2)
69
74
 
@@ -81,6 +86,12 @@ def _pathname(tempfile_or_path):
81
86
  tempfile_or_path,
82
87
  type(tempfile_or_path)))
83
88
 
89
+ # for skimage.measure.regionprops
90
+ def _width(region):
91
+ _,x1,_,x2 = region.bbox
92
+ return x2-x1
93
+
94
+
84
95
  def to_precision(x,p):
85
96
  """
86
97
  returns a string representation of x formatted with a precision of p
@@ -154,8 +165,8 @@ class Decoder:
154
165
  rec : Recording
155
166
  recording on which the decoder is working
156
167
 
157
- SN_ratio : float
158
- signal over noise ratio in dB.
168
+ effective_word_duration : float
169
+ duration of a word, influenced by ucontroller clock
159
170
 
160
171
  pulse_detection_level : float
161
172
  level used to detect sync pulse
@@ -176,7 +187,7 @@ class Decoder:
176
187
 
177
188
  """
178
189
 
179
- def __init__(self, aRec):
190
+ def __init__(self, aRec, do_plots):
180
191
  """
181
192
  Initialises Decoder
182
193
 
@@ -186,587 +197,307 @@ class Decoder:
186
197
 
187
198
  """
188
199
  self.rec = aRec
200
+ self.do_plots = do_plots
189
201
  self.clear_decoder()
190
202
 
203
+
191
204
  def clear_decoder(self):
192
205
  self.sound_data_extract = None
193
- self.cached_convolution_fit = {'sound_extract_position': None}
194
206
  self.pulse_detection_level = None
195
- self.silent_zone_indices = None
196
207
  self.detected_pulse_position = None
197
- self.estimated_pulse_position = None
198
-
199
- def set_sound_extract_and_sr(self, extract, s_r, where):
200
- self.sound_extract = extract
201
- self.samplerate = s_r
202
- self.sound_extract_position = where
203
- self.cached_convolution_fit = {'sound_extract_position': None}
204
- logger.debug('sound_extract set, samplerate %i location %i'%(s_r, where))
205
-
206
-
207
- # there's always at least one complete 0.5 silence interval in a 1.5 second signal
208
-
209
- def _get_envelope(self):
210
- """
211
- Compute self.sound_extract envelope, filtering its hilbert transform.
212
- Uses scipy.signal.hilbert() and scipy.signal.savgol_filter();
213
- window_length and polyorder savgol_filter() parameters values
214
- have been found by hit and miss on audio data.
215
-
216
- maybe:
217
- Values are roughly normalized: between 0 and approximately 1.0
218
-
219
- Returns
220
- -------
221
- numpy.ndarray the same length of self.recording.sound_extract
222
-
223
- """
224
- WINDOW_LENGTH, POLYORDER = (15, 3) # parameters found by experiment, hit and miss
225
- absolute_of_hilbert = np.abs(scipy.signal.hilbert(self.sound_extract))
226
- envelope = scipy.signal.savgol_filter(absolute_of_hilbert,
227
- WINDOW_LENGTH, POLYORDER)
228
- logger.debug('self.sound_extract envelope length %i samples'%len(envelope))
229
- return envelope
230
-
231
- def _get_signal_level(self):
232
- abs_signal = abs(self.sound_extract)
233
- return 2 * abs_signal.mean() # since 50% duty cycle
234
-
235
- def _get_approx_pulse_position(self):
236
- """
237
- Returns the estimated pulse position using the detected silent
238
- zone . The position is in samples number
239
- relative to extract beginning
240
- """
241
- # if self.detected_pulse_position:
242
- # logger.debug('returning detected value')
243
- # return self.detected_pulse_position
244
- if self.estimated_pulse_position:
245
- logger.debug('returning cached estimated value')
246
- return self.estimated_pulse_position ###############################
247
- _, silence_center_x = self._fit_triangular_signal_to_convoluted_env()
248
- # symbol_width_samples = 1e-3*SYMBOL_LENGTH
249
- self.estimated_pulse_position = silence_center_x + int(0.5*(
250
- 0.5 - 1e-3*SYMBOL_LENGTH)*self.samplerate)
251
- logger.debug('returning estimated value from silence mid position')
252
- return self.estimated_pulse_position
253
-
254
- def _get_pulse_position(self):
255
- # relative to extract beginning
256
- if self.detected_pulse_position:
257
- logger.debug('returning detected value')
258
- return self.detected_pulse_position
259
- return None
260
-
261
- def _get_pulse_detection_level(self):
262
- # return level at which the sync pulse will be detected
263
- if self.pulse_detection_level is None:
264
- silence_floor = self._get_silence_floor()
265
- # lower_BFSK_level = silence_floor
266
- # pulse_position = self._get_pulse_position()
267
- lower_BFSK_level = self._get_minimal_bfsk()
268
- value = math.sqrt(silence_floor * lower_BFSK_level)
269
- # value = OVER_NOISE_SYNC_DETECT_LEVEL * silence_floor
270
- logger.debug('setting pulse_detection_level to %f'%value)
271
- self.pulse_detection_level = value
272
- return value
273
- else:
274
- return self.pulse_detection_level
275
-
276
- def _get_square_convolution(self):
277
- """
278
- Compute self.sound_extract envelope convolution with a square signal of
279
- 0.5 second (+ SYMBOL_LENGTH) width, using numpy.convolve().
280
- Values are roughly normalized: between 0 and approximately 1.0
281
-
282
-
283
- Returns:
284
-
285
- #1 the normalized convolution, a numpy.ndarray shorter than
286
- self.sound_extract, see numpy.convolve(..., mode='valid')
287
-
288
- #2 a list of int, samples indexes where the convolution is computed
289
-
290
- """
291
- sqr_window_width = int((0.5 + SYMBOL_LENGTH/1e3)*self.samplerate) # in samples
292
- sqr_signal = np.ones(sqr_window_width,dtype=int)
293
- envelope = self._get_envelope()
294
- mean = envelope.mean()
295
- if mean: # in case of zero padding (? dont remember why)
296
- factor = 0.5/mean # since 50% duty cycle
297
- else:
298
- factor = 1
299
- normalized_envelope = factor*envelope
300
- convol = np.convolve(normalized_envelope, sqr_signal,
301
- mode='valid')/sqr_window_width
302
- start = int(sqr_window_width/2)
303
-
304
- x = range(start, len(convol) + start)
305
- return [*x], convol
306
-
307
- def _get_word_envelope(self):
308
- """
309
- Chop the signal envelope keeping the word region and smooth it over the
310
- longest BFSK period
311
- """
312
- SR = self.samplerate
313
- envelope = self._get_envelope()
314
- pulse_position = self._get_approx_pulse_position()
315
- samples_to_end = len(self.sound_extract) - pulse_position
316
- is_too_near_the_end = samples_to_end/SR < 0.5
317
- logger.debug('pulse_position is_too_near_the_end %s'%
318
- is_too_near_the_end)
319
- if is_too_near_the_end:
320
- pulse_position -= SR # one second sooner
321
- symbol_width_samples = 1e-3*SYMBOL_LENGTH*SR
322
- word_start = int(pulse_position + 3*symbol_width_samples)
323
- word_end = int(pulse_position + 0.5*SR)
324
- word_end -= int(2*symbol_width_samples) # slide to the left a little
325
- logger.debug('word start, end: %i %i (in extract)'%(
326
- word_start, word_end))
327
- logger.debug('word start, end: %i %i (in file)'%(
328
- word_start + self.sound_extract_position,
329
- word_end + self.sound_extract_position))
330
- w_envelope = envelope[word_start : word_end]
331
- word_envelope_truncated = word_end-word_start != len(w_envelope)
332
- logger.debug('w_envelope is sliced out of bounds: %s'%(
333
- str(word_envelope_truncated)))
334
- logger.debug('word envelope length %i samples %f secs'%(
335
- len(w_envelope), len(w_envelope)/SR))
336
- max_period = int(self.samplerate*max(1/F1,1/F2))
337
- logger.debug('max BFSK period %i in samples'%max_period)
338
- period_window = np.ones(max_period,dtype=int)/max_period
339
- # smooth over longest BFSK period
340
- return np.convolve(w_envelope, period_window, mode='same')
341
-
342
- def _get_minimal_bfsk(self):
343
- """
344
- because of non-flat frequency response, bfsk bits dont have the same
345
- amplitude. This returns the least of both by detecting a bimodal
346
- gaussian distribution
347
-
348
- """
349
- # w_envelope = self._get_word_envelope()
350
- # word_start = int(min_position + shift + 0.3*self.samplerate)
351
- # word = w_envelope[word_start : int(word_start + 0.4*self.samplerate)]
352
- word = self._get_word_envelope()
353
- # plt.plot(word)
354
- # plt.show()
355
- n = len(word)
356
- word = word.reshape(n, 1)
357
- gm = GaussianMixture(n_components=2, random_state=0).fit(word)
358
- bfsk_minimal_amplitude = min(gm.means_)
359
- logger.debug('bfsk_minimal_amplitude %f'%bfsk_minimal_amplitude)
360
- return bfsk_minimal_amplitude
361
-
362
- def _fit_triangular_signal_to_convoluted_env(self):
208
+
209
+ def set_sound_extract_and_sr(self, sound_extract, samplerate, sound_extract_position):
363
210
  """
364
- Try to fit a triangular signal to the envelope convoluted with a square
365
- signal to evaluate if audio is composed of 0.5 sec signal and 0.5 s
366
- silence. If so, the convolution is a triangular fct and a
367
- Levenberg–Marquardt fit is used to detect this occurence (lmfit).
368
- Results are cached in self.cached_convolution_fit alongside
369
- self.sound_extract_position for hit/miss checks.
370
-
211
+ Sets:
212
+ self.sound_extract -- mono data of short duration
213
+ self.samplerate -- in Hz
214
+ self.sound_extract_position -- position in the whole file
371
215
 
372
- Returns
373
- -------
374
- float
375
- chi_sqr from lmfit.minimize(), indicative of fit quality
376
- int
377
- position of triangular minimum (base of the v shape), this
378
- corresponds to the center of silent zone.
216
+ Computes and sets:
217
+ self.pulse_detection_level
218
+ self.sound_extract_one_bit
219
+ self.words_props (contains the sync pulse too)
379
220
 
221
+ Returns nothing
380
222
  """
381
- cached_value = self.cached_convolution_fit['sound_extract_position']
382
- logger.debug('cache hit? asked:%s cached: %s'%(
383
- self.sound_extract_position,
384
- cached_value))
385
- logger.debug('(sound_extract_position values)')
386
- if (CACHING and
387
- # cache_is_clean and
388
- cached_value and
389
- math.isclose(
390
- self.sound_extract_position,
391
- cached_value,
392
- abs_tol=0.1)):
393
- logger.debug('yes, fit values cached:')
394
- v1 = self.cached_convolution_fit['chi_square']
395
- v2 = self.cached_convolution_fit['minimum position']
396
- v2_file = v2 + self.sound_extract_position
397
- logger.debug('cached chi_sq: %s minimum position in file: %s'%
398
- (v1, v2_file))
399
- return (v1, v2) ####################################################
400
- # cached!
401
- x_shifted, convolution = self._get_square_convolution()
402
- # see numpy.convolve(..., mode='valid')
403
- x = np.arange(len(convolution))
404
- trig_params = lmfit.Parameters()
405
- trig_params.add(
406
- 'A', value=1, min=0, max=2
407
- )
408
- period0 = 2*self.samplerate
409
- trig_params.add(
410
- 'period', value=period0, min=0.9*period0,
411
- max=1.1*period0
412
- )
413
- trig_params.add(
414
- 'min_position', value=len(convolution)/2
415
- ) # at center
416
- def trig_wave(pars, x, signal_data=None):
417
- # looking for phase sx with a sin of 1 sec period and 0<y<1.0
418
- A = pars['A']
419
- p = pars['period']
420
- mp = pars['min_position']
421
- model = 2*A*arcsin(abs(sin((x - mp)*2*pi/p)))/pi
422
- if signal_data is None:
423
- return model ###################################################
424
- return model - signal_data
425
- fit_trig = lmfit.minimize(
426
- trig_wave, trig_params,
427
- args=(x,), kws={'signal_data': convolution}
428
- )
429
- chi_square = fit_trig.chisqr
430
- shift = x_shifted[0] # convolution is shorter than sound envelope
431
- min_position = int(fit_trig.params['min_position'].value) + shift
432
- logger.debug('chi_square %.1f minimum convolution position %i in file'%
433
- (chi_square, min_position + self.sound_extract_position))
434
- self.cached_convolution_fit['sound_extract_position'] = \
435
- self.sound_extract_position
436
- self.cached_convolution_fit['chi_square'] = chi_square
437
- self.cached_convolution_fit['minimum position'] = min_position
438
-
439
- return chi_square, min_position + shift
223
+ logger.debug('sound_extract: %s, samplerate: %s Hz, sound_extract_position %s'%(
224
+ sound_extract, samplerate, sound_extract_position))
225
+ if len(sound_extract) == 0:
226
+ logger.error('sound extract is empty, is sound track duration OK?')
227
+ raise Exception('sound extract is empty, is sound track duration OK?')
228
+ self.sound_extract_position = sound_extract_position
229
+ self.samplerate = samplerate
230
+ self.sound_extract = sound_extract
231
+ self.pulse_detection_level = np.std(sound_extract)/4
232
+ logger.debug('pulse_detection_level %f'%self.pulse_detection_level)
233
+ bits = np.abs(sound_extract)>self.pulse_detection_level
234
+ N_ones = round(1.5*SYMBOL_LENGTH*1e-3*samplerate) # so it includes sync pulse
235
+ self.sound_extract_one_bit = closing(bits, np.ones(N_ones))
236
+ if self.do_plots:
237
+ self._plot_extract()
238
+ logger.debug('sound_extract_one_bit len %i'%len(self.sound_extract_one_bit))
239
+ self.words_props = regionprops(label(np.array(2*[self.sound_extract_one_bit]))) # new
440
240
 
441
241
  def extract_seems_TicTacCode(self):
442
- """margin
443
- evaluate if sound data is half signal, half silence
444
- no test is done on frequency components nor BFSK modulation.
445
-
446
- Returns
447
- -------
448
- True if sound seems TicTacCode
449
-
450
- """
451
- chi_square, _ = self._fit_triangular_signal_to_convoluted_env()
452
- seems_TicTacCode = chi_square < 200 # good fit so, yes
453
- logger.debug('seems TicTacCode: %s'%seems_TicTacCode)
454
- return seems_TicTacCode
242
+ """
243
+ Determines if signal in sound_extract seems to be TTC.
455
244
 
456
- def _get_silent_zone_indices(self):
457
- """
458
- Returns silent zone boundary positions relative to the start
459
- of self.sound_extract.
245
+ Uses the conditions below:
460
246
 
461
- Returns
462
- -------
463
- left_window_boundary : int
464
- left indice.
465
- right_window_boundary : int
466
- right indice.
247
+ Extract duration is 1.143 s.
248
+ In self.word_props (list of morphology.regionprops):
249
+ if one region, duration should be in [0.499 0.512] sec
250
+ if two regions, total duration should be in [0.50 0.655]
467
251
 
252
+ Returns True if self.sound_data_extract seems TicTacCode
468
253
  """
469
- if self.silent_zone_indices:
470
- return self.silent_zone_indices ####################################
471
- _, silence_center_position = \
472
- self._fit_triangular_signal_to_convoluted_env()
473
- srate = self.samplerate
474
- half_window = int(SAFE_SILENCE_WINDOW_WIDTH * 1e-3 * srate/2)
475
- left_window_boundary = silence_center_position - half_window
476
- right_window_boundary = silence_center_position + half_window
477
- # margin = 0.75 * srate
478
- values = np.array([left_window_boundary, right_window_boundary,
479
- silence_center_position])
480
- values += self.sound_extract_position # samples pos in file
481
- logger.debug('silent zone, left: %i, right %i, center %i'%tuple(values))
482
- self.silent_zone_indices = (left_window_boundary, right_window_boundary)
483
- return self.silent_zone_indices
484
-
485
- def _get_silence_floor(self):
486
- # analyses the 0.5 silence zone
487
- start_silent_zone, end_silent_zone = self._get_silent_zone_indices()
488
- signal = self.sound_extract
489
- silent_signal = signal[start_silent_zone:end_silent_zone]
490
- max_value = 1.001*np.abs(silent_signal).max() # a little headroom
491
- max_value = 0 # should toggle this with a CLI option
492
- five_sigmas = 5 * silent_signal.std()
493
- return max(max_value, five_sigmas) # if guassian, five sigmas will do it
494
-
495
- def make_silence_analysis_plot(self, title=None, filename=None):
496
- # save figure in filename if set, otherwise
497
- # start an interactive plot, title is for matplotlib
498
- pulse_pos_in_file = self._get_approx_pulse_position()
499
- pulse_pos_in_file += self.sound_extract_position
500
- pulse_position_sec = pulse_pos_in_file/self.samplerate
501
- duration_in_sec = self.rec.get_duration()
502
- if pulse_position_sec > duration_in_sec/2:
503
- pulse_position_sec -= duration_in_sec
504
- title = 'Silence analysis around %.2f s'%(pulse_position_sec)
505
- logger.debug('make_silence_analysis_plot(title=%s, filename=%s)'%(
506
- title, filename))
507
- start_silent_zone, end_silent_zone = self._get_silent_zone_indices()
508
- signal = self.sound_extract
509
- x_signal = range(len(signal))
510
- x_convolution, convolution = self._get_square_convolution()
511
- scaled_convo = self._get_signal_level()*convolution
512
- # since 0 < convolution < 1
513
- trig_level = self._get_pulse_detection_level()
514
- sound_extract_position = self.sound_extract_position
515
- def x2f(nx):
516
- return nx + sound_extract_position
517
- def f2x(nf):
518
- return nf - sound_extract_position
254
+ failing_comment = '' # used as a flag
255
+ props = self.words_props
256
+ if len(props) not in [1,2]:
257
+ failing_comment = 'len(props) not in [1,2]: %i'%len(props)
258
+ if len(props) == 1:
259
+ w = _width(props[0])/self.samplerate
260
+ # self.effective_word_duration = w
261
+ # logger.debug('effective_word_duration %f (one region)'%w)
262
+ if not 0.499 < w < 0.512: # TODO: move as TOP OF FILE PARAMS
263
+ failing_comment = '_width %f not in [0.499 0.512]'%w
264
+ else: # 2 regions
265
+ widths = [_width(p)/self.samplerate for p in props] # in sec
266
+ total_w = sum(widths)
267
+ # extra_window_duration = SOUND_EXTRACT_LENGTH - 1
268
+ # eff_w = total_w - extra_window_duration
269
+ # logger.debug('effective_word_duration %f (two regions)'%eff_w)
270
+ if not 0.5 < total_w < 0.655:
271
+ failing_comment = 'two regions duration %f not in [0.50 0.655]\n%s'%(total_w, widths)
272
+ # fig, ax = plt.subplots()
273
+ # p(ax, sound_extract_one_bit)
274
+ logger.debug('failing_comment: %s'%(
275
+ 'none' if failing_comment=='' else failing_comment))
276
+ return failing_comment == '' # no comment = extract seems TicTacCode
277
+
278
+ def _plot_extract(self):
519
279
  fig, ax = plt.subplots()
520
- plt.title(title)
521
- secax = ax.secondary_xaxis('top', functions=(x2f, f2x))
522
- # secax.set_xlabel('position in file')
523
- # ax.set_xlabel('position in extract')
524
- yt = ax.get_yaxis_transform()
280
+ ax.plot(self.sound_extract, marker='o', markersize='1',
281
+ linewidth=1.5,alpha=0.3, color='blue' )
282
+ ax.plot(self.sound_extract_one_bit*np.max(np.abs(self.sound_extract)),
283
+ marker='o', markersize='1',
284
+ linewidth=1.5,alpha=0.3,color='red')
525
285
  xt = ax.get_xaxis_transform()
526
- _, silence_center_x = self._fit_triangular_signal_to_convoluted_env()
527
- # symbol_width_samples = 1e-3*SYMBOL_LENGTH
528
- # pulse_relative_pos = int(0.5*(0.5 - 1e-3*SYMBOL_LENGTH)*self.samplerate)
529
- approx_pulse_x = self._get_approx_pulse_position()
530
- ax.vlines(
531
- silence_center_x, 0.4, 0.6,
532
- transform=xt, linewidth=1, colors='black'
533
- )
534
- ax.vlines(
535
- approx_pulse_x, 0.1, 0.9,
536
- transform=xt, linewidth=1, colors='yellow'
537
- )
538
- bfsk_min = self._get_minimal_bfsk()
539
- ax.hlines(
540
- bfsk_min, 0, 1,
541
- transform=yt, linewidth=1, colors='red'
542
- )
543
- ax.hlines(
544
- trig_level, 0, 1,
545
- transform=yt, linewidth=1, colors='blue'
546
- )
547
- ax.hlines(
548
- -trig_level, 0, 1,
549
- transform=yt, linewidth=1, colors='blue'
550
- )
551
- ax.hlines(
552
- 0, 0, 1,
553
- transform=yt, linewidth=0.5, colors='black'
554
- )
555
- # plt.title(title)
286
+ yt = ax.get_yaxis_transform()
287
+ ax.hlines(self.pulse_detection_level, 0, 1,
288
+ transform=yt, alpha=0.3,
289
+ linewidth=2, colors='green')
556
290
  custom_lines = [
557
- Line2D([0], [0], color='black', lw=2),
558
291
  Line2D([0], [0], color='green', lw=2),
559
292
  Line2D([0], [0], color='blue', lw=2),
560
293
  Line2D([0], [0], color='red', lw=2),
561
- Line2D([0], [0], color='yellow', lw=2),
562
294
  ]
563
295
  ax.legend(
564
296
  custom_lines,
565
- 'sqr convolution,silence zone,pulse level,lower FSK ampl., approx puls.'.split(','),
297
+ 'detection level, signal, detected region'.split(','),
566
298
  loc='lower right')
567
- ax.plot(
568
- x_signal, signal,
569
- marker='o', markersize='1',
570
- linewidth=0.3, color='purple', alpha=0.3)
571
- ax.axvspan(
572
- start_silent_zone, end_silent_zone,
573
- alpha=0.1, color='green')
574
- ax.plot(
575
- x_convolution, scaled_convo,
576
- marker='.', markersize='0.2',
577
- linestyle='None', color='black', alpha=1)
578
- # linewidth=0.3, linestyle='None', color='black', alpha=0.3)
579
- # ax.set_xlabel('Decoder.sound_extract samples')
580
- if filename == None:
581
- plt.show()
582
- else:
583
- logger.debug('saving silence_analysis_plot to %s'%filename)
584
- plt.savefig(
585
- filename,
586
- format="png")
587
- plt.close()
588
-
589
- def _detect_sync_pulse_position(self):
590
- """
591
- Determines noise level during silence period and use it to detect the
592
- sync pulse position. Computes SN_ratio and stores it. Start searching
593
- around end of silent zone. Adjustment are made so a complete 0.5
594
- second signal is at the right of the starting search position so a
595
- complete 0.5 s word is available for decoding.
299
+ ax.set_title('Finding word and sync pulse')
300
+ plt.show()
596
301
 
597
- Returns the pulse position relative to the extract beginning.
598
-
599
- Sets self.detected_pulse_position to the returned value.
302
+ def get_time_in_sound_extract(self):
303
+ """
304
+ Tries to decode time present in self.sound_extract, if successfull
305
+ return a time dict, eg:{'version': 0, 'seconds':
306
+ 44, 'minutes': 57, 'hours': 19,
307
+ 'day': 1, 'month': 3, 'year offset': 1,
308
+ 'pulse at': 670451.2217 } otherwise return None
600
309
  """
601
- pulse_detection_level = self._get_pulse_detection_level()
602
- abs_signal = abs(self.sound_extract)
603
- mean_during_word = 2*abs_signal.mean()
604
- self.SN_ratio = 20*math.log10(mean_during_word/pulse_detection_level)
605
- logger.debug('SN ratio: %f dB'%(self.SN_ratio))
606
- search_pulse_start_point = self._get_approx_pulse_position()
607
- search_pulse_start_point -= 3*SYMBOL_LENGTH*1e-3*self.samplerate
608
- search_pulse_start_point = int(search_pulse_start_point)
609
- # search_pulse_start_point = end_silent_zone
610
- logger.debug('_get_pulse_position %i as starting point'%search_pulse_start_point)
611
- distance_to_end = len(self.sound_extract) - search_pulse_start_point
612
- time_to_end = distance_to_end/self.samplerate
613
- logger.debug('time_to_end after search_pulse_start_point %f sec'%
614
- time_to_end)
615
- if time_to_end < 0.5 + SYMBOL_LENGTH*1e-3: # sec
616
- logger.debug('search_pulse_start_point too near the end')
617
- # go back one second
618
- search_pulse_start_point -= self.samplerate
619
- logger.debug('new starting point %i'%search_pulse_start_point)
310
+ pulse_detected = self._detect_sync_pulse_position()
311
+ if not pulse_detected:
312
+ return None
313
+ symbols_data = self._get_symbols_data()
314
+ frequencies = [self._get_main_frequency(data_slice)
315
+ for data_slice in symbols_data ]
316
+ logger.debug('found frequencies %s'%frequencies)
317
+ def _get_bit_from_freq(freq):
318
+ mid_FSK = 0.5*(F1 + F2)
319
+ return '1' if freq > mid_FSK else '0'
320
+ bits = [_get_bit_from_freq(f) for f in frequencies]
321
+ bits_string = ''.join(bits)
322
+ logger.debug('giving bits: LSB %s MSB'%bits_string)
323
+
324
+ def _values_from_bits(bits):
325
+ word_payload_bits_positions = {
326
+ # start, finish (excluded)
327
+ 'version':(0,3), # 3 bits
328
+ 'seconds':(3,9), # 6 bits
329
+ 'minutes':(9,15),
330
+ 'hours':(15,20),
331
+ 'day':(20,25),
332
+ 'month':(25,29),
333
+ 'year offset':(29,34),
334
+ }
335
+ binary_words = { key : bits[slice(*value)]
336
+ for key, value
337
+ in word_payload_bits_positions.items()
338
+ }
339
+ int_values = { key : int(''.join(reversed(val)),2)
340
+ for key, val in binary_words.items()
341
+ }
342
+ return int_values
343
+ time_values = _values_from_bits(bits_string)
344
+ logger.debug(' decoded time %s'%time_values)
345
+ sync_pos_in_file = self.detected_pulse_position + \
346
+ self.sound_extract_position
347
+ time_values['pulse at'] = sync_pos_in_file
348
+ return time_values
349
+
350
+ def _detect_sync_pulse_position(self):
351
+ # sets self.detected_pulse_position, relative to sound_extract
352
+ #
353
+ regions = self.words_props # contains the sync pulse too
354
+ # len(self.words_props) should be 1 or 2 for vallid TTC
355
+ logger.debug('len() of words_props: %i'%len(self.words_props))
356
+ whole_region = [p for p in regions if 0.499 < _width(p)/self.samplerate < 0.512]
357
+ logger.debug('region widths %s'%[_width(p)/self.samplerate for p in regions])
358
+ logger.debug('number of whole_region %i'%len(whole_region))
359
+ if len(regions) == 1 and len(whole_region) != 1:
360
+ # oops
361
+ logger.debug('len(regions) == 1 and len(whole_region) != 1, failed')
362
+ return False #######################################################
363
+ if len(whole_region) > 1:
364
+ print('error in _detect_sync_pulse_position: len(whole_region) > 1 ')
365
+ return False #######################################################
366
+ if len(whole_region) == 1:
367
+ # sync pulse at the begining of this one
368
+ _, spike, _, _ = whole_region[0].bbox
620
369
  else:
621
- logger.debug('ok, ample room for 0.5 s word')
622
- search_pulse_start_point = max(search_pulse_start_point, 0)
623
- logger.debug('search_pulse_start_point: %i in extract'%
624
- search_pulse_start_point)
625
- abs_signal_after_silence = abs_signal[search_pulse_start_point:]
626
- # here the real searching with numpy.argmax()
627
- first_point = \
628
- np.argmax(abs_signal_after_silence > pulse_detection_level)
629
- first_point += search_pulse_start_point
630
- logger.debug('found sync pulse at %i in extract'%first_point)
631
- logger.debug('found sync pulse at %i in file'%(first_point
632
- + self.sound_extract_position))
633
- distance_to_end = len(self.sound_extract) - first_point
634
- time_to_end = distance_to_end/self.samplerate
635
- logger.debug('time_to_end after detected pulse %f sec'%
636
- time_to_end)
637
- if time_to_end < 0.5 + SYMBOL_LENGTH*1e-3: # sec
638
- logger.debug('detected_pulse_position too near the end')
639
- logger.debug('back one second')
640
- first_point -= self.samplerate
641
- logger.debug('new sync pulse at %i in extract'%first_point)
642
- logger.debug('new sync pulse at %i in file'%first_point)
370
+ # whole_region is [] (all fractionnal) and
371
+ # sync pulse at the begining of the 2nd region
372
+ _, spike, _, _ = regions[1].bbox
373
+ # but check there is still enough place for ten bits:
374
+ # 6 for secs + 3 for revision + blanck after sync
375
+ minimum_samples = int(self.samplerate*10*SYMBOL_LENGTH*1e-3)
376
+ whats_left = len(self.sound_extract) - spike
377
+ if whats_left < minimum_samples:
378
+ spike -= self.samplerate
379
+ # else: stay there, will decode seconds in whats_left
380
+ half_symbol_width = int(0.5*1e-3*SYMBOL_LENGTH*self.samplerate) # samples
381
+ left, right = (spike - half_symbol_width, spike+half_symbol_width)
382
+ spike_data = self.sound_extract[left:right]
383
+ biggest_positive = np.max(spike_data)
384
+ biggest_negative = np.min(spike_data)
385
+ if abs(biggest_negative) > biggest_positive:
386
+ # flip
387
+ spike_data = -1 * spike_data
388
+ def fit_line_until_negative():
389
+ import numpy as np
390
+ start = np.argmax(spike_data)
391
+ xs = [start]
392
+ ys = [spike_data[start]]
393
+ i = 1
394
+ while spike_data[start - i] > 0 and start - i >= 0:
395
+ xs.append(start - i)
396
+ ys.append(spike_data[start - i])
397
+ i += 1
398
+ # ax.scatter(xs, ys)
399
+ import numpy as np
400
+ coeff = np.polyfit(xs, ys, 1)
401
+ m, b = coeff
402
+ zero = int(-b/m)
403
+ # check if data is from USB audio and tweak
404
+ y_fit = np.poly1d(coeff)(xs)
405
+ err = abs(np.sum(np.abs(y_fit-ys))/np.mean(ys))
406
+ logger.debug('fit error for line in ramp: %f'%err)
407
+ if err < 0.01: #good fit so not analog
408
+ zero += 1
409
+ return zero
410
+ sync_sample = fit_line_until_negative() + left
411
+ logger.debug('sync pulse found at %i in extract, %i in file'%(
412
+ sync_sample, sync_sample + self.sound_extract_position))
413
+ self.detected_pulse_position = sync_sample
414
+ return True
415
+
416
+ def _get_symbols_data(self):
417
+ # part of extract AFTER sync pulse
418
+ whats_left = len(self.sound_extract) - self.detected_pulse_position # in samples
419
+ whats_left /= self.samplerate # in sec
420
+ whole_word_is_in_extr = whats_left > 0.512
421
+ if whole_word_is_in_extr:
422
+ # one region
423
+ logger.debug('word is in one sole region')
424
+ length_needed = round(0.5*self.samplerate)
425
+ length_needed += round(self.samplerate*SYMBOL_LENGTH*1e-3)
426
+ whole_word = self.sound_extract[self.detected_pulse_position:
427
+ self.detected_pulse_position + length_needed]
643
428
  else:
644
- logger.debug('ok, ample room for 0.5 s word after detected pulse')
645
- self.detected_pulse_position = first_point
646
- return first_point
647
-
648
- def _get_word_width_parameters(self):
649
- """
650
- Returns the parameters used to find and demodulate 2FSK word:
651
- presumed_symbol_length
652
- word_width_threshold
653
- search_end_position
654
- """
655
- abs_signal = abs(self.sound_extract)
656
- pulse_position = self._get_pulse_position()
657
- bfsk_min = self._get_minimal_bfsk()
658
- params = {'word_width_threshold': 0.8*bfsk_min}
659
- sr = self.samplerate
660
- presumed_symbol_length = SYMBOL_LENGTH*1e-3*sr
661
- # in ms, now in samples
662
- presumed_word_width = presumed_symbol_length * N_SYMBOLS_SAMD21
663
- # word includes sync pulse
664
- params['search_end_position'] = \
665
- int(presumed_symbol_length + pulse_position + \
666
- presumed_word_width) + 1700 # samples for headroom
667
- params['presumed_symbol_length'] = presumed_symbol_length
668
- return params
669
-
670
- def _get_BFSK_word_boundaries(self):
671
- n_bits = N_SYMBOLS_SAMD21 - 1
672
- sr = self.samplerate
673
- wwp = self._get_word_width_parameters()
674
- pulse_position = self._get_pulse_position()
675
- # search_start_position = wwp['search_start_position']
676
- search_end_position = wwp['search_end_position']
677
- word_width_threshold = wwp['word_width_threshold']
678
- word_extract = self.sound_extract[pulse_position :
679
- search_end_position]
680
- logger.debug('word left boundary search position: %i relative to extract'%
681
- (search_end_position))
682
- logger.debug('extract starting at %i in file (Decoder.sound_extract_position)'%
683
- self.sound_extract_position)
684
- flipped_extract = np.flip(np.abs(word_extract))
685
- right_boundary = len(word_extract) - \
686
- np.argmax(flipped_extract > word_width_threshold) + \
687
- pulse_position
688
- # left_boundary = np.argmax(
689
- # np.abs(word_extract) > word_width_threshold) + \
690
- # search_start_position
691
- left_boundary = pulse_position
692
- left_in_file = left_boundary + self.sound_extract_position
693
- right_in_file = right_boundary + self.sound_extract_position
694
- logger.debug('word boundaries: %i and %i in file'%
695
- (left_in_file, right_in_file))
696
- eff_symbol_length = 1e3*(right_boundary-left_boundary)/(n_bits*sr)
697
- logger.debug('effective symbol length %.4f ms '%
698
- eff_symbol_length)
699
- logger.debug('presumed symbol length %.4f ms '%SYMBOL_LENGTH)
700
- relative_error = (eff_symbol_length-SYMBOL_LENGTH)/SYMBOL_LENGTH
701
- status = True
702
- if relative_error > SYMBOL_LENGTH_TOLERANCE:
703
- logger.error(
704
- 'actual symbol length differs too much: %.2f vs %.2f ms'%
705
- (eff_symbol_length, SYMBOL_LENGTH))
706
- # return None, None
707
- status = False
708
- logger.debug(' relative discrepancy %.4f%%'%(abs(100*relative_error)))
709
- return status, left_boundary, right_boundary
710
-
711
- def _get_BFSK_symbols_boundaries(self):
712
- # returns indices of start of each slice and boundaries
713
- pulse_position = self._get_pulse_position()
714
- boundaries_OK, left_boundary, right_boundary = \
715
- self._get_BFSK_word_boundaries()
716
- if left_boundary is None:
717
- return None, None, None ############################################
718
- symbol_width_samples = \
719
- float(right_boundary - left_boundary)/N_SYMBOLS_SAMD21
720
- symbol_positions = symbol_width_samples * \
721
- np.arange(float(0), float(N_SYMBOLS_SAMD21 + 1)) + \
722
- pulse_position
723
- int_symb_positions = symbol_positions.round().astype(int)
724
- logger.debug('%i symbol positions %s samples in file'%
725
- (
726
- len(int_symb_positions),
727
- int_symb_positions + self.sound_extract_position)
728
- )
729
- return int_symb_positions[:-1], left_boundary, right_boundary
730
-
731
- def _values_from_bits(self, bits):
732
- word_payload_bits_positions = {
733
- 'version':(0,2),
734
- 'clock source':(2,3),
735
- 'seconds':(3,9),
736
- 'minutes':(9,15),
737
- 'hours':(15,20),
738
- 'day':(20,25),
739
- 'month':(25,29),
740
- 'year offset':(29,34),
741
- }
742
- binary_words = { key : bits[slice(*value)]
743
- for key, value
744
- in word_payload_bits_positions.items()
745
- }
746
- int_values = { key : self._get_int_from_binary_str(val)
747
- for key, val in binary_words.items()
748
- }
749
- logger.debug(' Demodulated values %s'%int_values)
750
- return int_values
751
-
752
- def _slice_sound_extract(self, symbols_indices):
753
- indices_left_shifted = deque(list(symbols_indices))
754
- indices_left_shifted.rotate(-1)
755
- all_intervals = list(zip(
756
- symbols_indices,
757
- indices_left_shifted
758
- ))
759
- word_intervals = all_intervals[1:-1]
760
- # [0, 11, 23, 31, 45] => [(11, 23), (23, 31), (31, 45)]
761
- logger.debug('slicing intervals, word_intervals = %s'%
762
- word_intervals)
763
- # skip sample after pulse, start at BFSK word
764
- filtered_sound_extract = self._band_pass_filter(self.sound_extract)
765
- slices = [filtered_sound_extract[slice(*pair)]
766
- for pair in word_intervals]
767
- np.set_printoptions(threshold=5)
768
- # logger.debug('data slices: \n%s'%pprint.pformat(slices))
769
- # raise Exception
429
+ # Two regions.
430
+ logger.debug('word is in two regions, will wrap past seconds')
431
+ # Consistency check: if not whole_word_is_in_extr
432
+ # check has been done so seconds are encoded in what s left
433
+ minimum_samples = round(self.samplerate*10*SYMBOL_LENGTH*1e-3)
434
+ if whats_left*self.samplerate < minimum_samples:
435
+ print('bug in _get_data_symbol():')
436
+ print(' whats_left*self.samplerate < minimum_samples')
437
+ # Should now build a whole 0.5 sec word by joining remaining data
438
+ # from previous second beep
439
+ left_piece = self.sound_extract[self.detected_pulse_position:]
440
+ one_second_before_idx = round(len(self.sound_extract) - self.samplerate)
441
+ length_needed = round(0.5*self.samplerate - len(left_piece))
442
+ length_needed += round(self.samplerate*SYMBOL_LENGTH*1e-3)
443
+ right_piece = self.sound_extract[one_second_before_idx:
444
+ one_second_before_idx + length_needed]
445
+ whole_word = np.concatenate((left_piece, right_piece))
446
+ logger.debug('two chunks lengths: %i %i samples'%(len(left_piece),
447
+ len(right_piece)))
448
+ # search for word start (some jitter because of Teensy Audio Lib)
449
+ symbol_length = round(self.samplerate*SYMBOL_LENGTH*1e-3)
450
+ start = round(0.5*symbol_length) # half symbol
451
+ end = start + symbol_length
452
+ word_begining = whole_word[start:]
453
+ # word_one_bit = np.abs(word_begining)>self.pulse_detection_level
454
+ # N_ones = round(1.5*SYMBOL_LENGTH*1e-3*self.samplerate) # so it includes sync pulse
455
+ # word_one_bit = closing(word_one_bit, np.ones(N_ones))
456
+ gt_detection_level = np.argwhere(np.abs(word_begining)>self.pulse_detection_level)
457
+ # print(gt_detection_level)
458
+ # plt.plot(word_one_bit)
459
+ # plt.plot(word_begining/abs(np.max(word_begining)))
460
+ # plt.show()
461
+ word_start = gt_detection_level[0][0]
462
+ word_end = gt_detection_level[-1][0]
463
+ self.effective_word_duration = (word_end - word_start)/self.samplerate
464
+ logger.debug('effective_word_duration %f s'%self.effective_word_duration)
465
+ uCTRLR_error = self.effective_word_duration/((N_SYMBOLS -1)*SYMBOL_LENGTH*1e-3)
466
+ logger.debug('uCTRLR_error %f (time ratio)'%uCTRLR_error)
467
+ word_start += start # relative to Decoder extract
468
+ # check if gap is indeed less than TEENSY_MAX_LAG
469
+ silence_length = word_start
470
+ gap = silence_length - symbol_length
471
+ relative_gap = gap/(TEENSY_MAX_LAG*self.samplerate)
472
+ logger.debug('Audio update() gap between sync pulse and word start: ')
473
+ logger.debug('%.2f ms (max value %.2f)'%(1e3*gap/self.samplerate,
474
+ 1e3*TEENSY_MAX_LAG))
475
+ logger.debug('relative audio_block gap %.2f'%(relative_gap))
476
+ if relative_gap > 1:
477
+ print('bug with relative_gap')
478
+ symbol_width_samples_theor = self.samplerate*SYMBOL_LENGTH*1e-3
479
+ symbol_width_samples_eff = self.effective_word_duration * \
480
+ self.samplerate/(N_SYMBOLS - 1)
481
+ logger.debug('symbol width %i theo; %i effective (samples)'%(
482
+ symbol_width_samples_theor,
483
+ symbol_width_samples_eff))
484
+ symbol_positions = symbol_width_samples_eff * \
485
+ np.arange(float(0), float(N_SYMBOLS - 1)) + word_start
486
+ # symbols_indices contains 34 start of symbols (samples)
487
+ symbols_indices = symbol_positions.round().astype(int)
488
+ if self.do_plots:
489
+ fig, ax = plt.subplots()
490
+ ax.plot(whole_word, marker='o', markersize='1',
491
+ linewidth=1.5,alpha=0.3, color='blue' )
492
+ xt = ax.get_xaxis_transform()
493
+ for x in symbols_indices:
494
+ ax.vlines(x, 0, 1,
495
+ transform=xt,
496
+ linewidth=0.6, colors='green')
497
+ ax.set_title('Slicing the 34 bits word:')
498
+ plt.show()
499
+ slice_width = round(SYMBOL_LENGTH*1e-3*self.samplerate)
500
+ slices = [whole_word[i:i+slice_width] for i in symbols_indices]
770
501
  return slices
771
502
 
772
503
  def _get_main_frequency(self, symbol_data):
@@ -777,195 +508,6 @@ class Decoder:
777
508
  freq_in_hertz = abs(freq * self.samplerate)
778
509
  return int(round(freq_in_hertz))
779
510
 
780
- # def _get_bit_from_freq(self, freq):
781
- # if math.isclose(freq, F1, abs_tol=FSK_TOLERANCE):
782
- # return '0'
783
- # if math.isclose(freq, F2, abs_tol=FSK_TOLERANCE):
784
- # return '1'
785
- # else:
786
- # return None
787
-
788
- def _get_bit_from_freq(self, freq):
789
- mid_FSK = 0.5*(F1 + F2)
790
- return '1' if freq > mid_FSK else '0'
791
-
792
- def _get_int_from_binary_str(self, string_of_01s):
793
- return int(''.join(reversed(string_of_01s)),2)
794
- # LSB is leftmost in TicTacCode
795
-
796
- def _demod_values_are_OK(self, values_dict):
797
- # TODO: use _get_timedate_from_dict rather (catching any ValueError)
798
- ranges = {
799
- 'seconds': range(60),
800
- 'minutes': range(60),
801
- 'hours': range(24),
802
- 'day': range(1,32), # 32 ?
803
- 'month': range(1,13),
804
- }
805
- for key in ranges:
806
- val = values_dict[key]
807
- ok = val in ranges[key]
808
- logger.debug(
809
- 'checking range for %s: %i, Ok? %s'%(key, val, ok))
810
- if not ok:
811
- logger.error('demodulated value is out of range')
812
- return False
813
- return True
814
-
815
- def _plot_slices(self, sync_pulse, symbols_indices, word_lft, word_rght,
816
- title=None, filename=None):
817
- # save figure in filename if set, otherwise
818
- # start an interactive plot, title is for matplotlib
819
- signal = self.sound_extract
820
- # signal = self._band_pass_filter(signal)
821
- start = self.sound_extract_position
822
- x_signal_in_file = range(
823
- start,
824
- start + len(signal)
825
- )
826
- wwp = self._get_word_width_parameters()
827
- start_silent_zone, end_silent_zone = self._get_silent_zone_indices()
828
- search_end_position = wwp['search_end_position'] + start
829
- logger.debug('doing slice plot')
830
- fig, ax = plt.subplots()
831
- plt.title(title)
832
- ax.plot(
833
- x_signal_in_file, signal,
834
- marker='.', markersize='1',
835
- linewidth=0.3, color='purple', alpha=0.3)
836
- yt = ax.get_yaxis_transform()
837
- ax.hlines(
838
- wwp['word_width_threshold'], 0, 1,
839
- transform=yt, linewidth=0.6, colors='green')
840
- ax.hlines(
841
- 0, 0, 1,
842
- transform=yt, linewidth=0.6, colors='black')
843
- ax.hlines(
844
- -wwp['word_width_threshold'], 0, 1,
845
- transform=yt, linewidth=0.6, colors='green')
846
- pulse_level = self._get_pulse_detection_level()
847
- ax.hlines(
848
- pulse_level, 0, 1,
849
- transform=yt, linewidth=0.6, colors='blue')
850
- ax.hlines(
851
- -pulse_level, 0, 1,
852
- transform=yt, linewidth=0.6, colors='blue')
853
- xt = ax.get_xaxis_transform()
854
- # ax.vlines(
855
- # search_start_position,
856
- # 0, 1, transform=xt, linewidth=0.6, colors='blue')
857
- ax.vlines(
858
- search_end_position,
859
- 0, 1, transform=xt, linewidth=0.6, colors='blue')
860
- ax.plot(
861
- [sync_pulse + start], [0],
862
- marker='D', markersize='7',
863
- linewidth=0.3, color='blue', alpha=0.3)
864
- ax.plot(
865
- [start_silent_zone + start], [0],
866
- marker='>', markersize='10',
867
- linewidth=0.3, color='green', alpha=0.3)
868
- ax.plot(
869
- [end_silent_zone + start], [0],
870
- marker='<', markersize='10',
871
- linewidth=0.3, color='green', alpha=0.3)
872
- boundaries_OK, word_lft, word_rght = \
873
- self._get_BFSK_word_boundaries()
874
- ax.vlines(
875
- word_lft + start, 0, 1,
876
- transform=ax.get_xaxis_transform(),
877
- linewidth=0.6, colors='red')
878
- ax.vlines(
879
- word_rght + start, 0, 1,
880
- transform=ax.get_xaxis_transform(),
881
- linewidth=0.6, colors='red')
882
- for x in symbols_indices + self.sound_extract_position:
883
- ax.vlines(
884
- x, 0, 1,
885
- transform=ax.get_xaxis_transform(),
886
- linewidth=0.3, colors='green')
887
- ax.set_xlabel(
888
- 'samples in file')
889
- plt.xlim(
890
- [sync_pulse - 300 + start, wwp['search_end_position'] + 400 + start])
891
- if filename == None:
892
- plt.show()
893
- else:
894
- plt.ylim(
895
- [-1.5*wwp['word_width_threshold'],
896
- 1.1*signal.max()])
897
- height = 1000
898
- plt.savefig(
899
- filename,
900
- format='png',
901
- dpi=height/fig.get_size_inches()[1])
902
- plt.close()
903
- logger.debug('done slice plot')
904
-
905
- def _band_pass_filter(self, data):
906
- # return filtered data
907
- def _bandpass(data: np.ndarray, edges: list[float], sample_rate: float, poles: int = 5):
908
- sos = scipy.signal.butter(poles, edges, 'bandpass', fs=sample_rate, output='sos')
909
- filtered_data = scipy.signal.sosfiltfilt(sos, data)
910
- return filtered_data
911
- sample_rate = self.samplerate
912
- times = np.arange(len(data))/sample_rate
913
- return _bandpass(data, [BPF_LOW_FRQ, BPF_HIGH_FRQ], sample_rate)
914
-
915
- def get_time_in_sound_extract(self, plots):
916
- if self.sound_extract is None:
917
- return None ########################################################
918
- if plots:
919
- self.make_silence_analysis_plot()
920
- pulse_position = self._detect_sync_pulse_position()
921
- pulse_pos_in_file = pulse_position + self.sound_extract_position
922
- pulse_position_sec = pulse_pos_in_file/self.samplerate
923
- logger.debug('found sync pulse at sample %i in file'%pulse_pos_in_file)
924
- symbols_indices, word_lft, word_rght = \
925
- self._get_BFSK_symbols_boundaries()
926
- if plots:
927
- title = 'Bit slicing at %s, %.2f s'%(pulse_pos_in_file,
928
- pulse_position_sec)
929
- # self.make_silence_analysis_plot()
930
- logger.debug('calling _plot_slices()')
931
- self._plot_slices(pulse_position, symbols_indices, word_lft,
932
- word_rght, title)
933
- if symbols_indices is None:
934
- return None ########################################################
935
- sliced_data = self._slice_sound_extract(symbols_indices)
936
- frequencies = [self._get_main_frequency(data_slice)
937
- for data_slice
938
- in sliced_data
939
- ]
940
- logger.debug('frequencies = %s'%frequencies)
941
- sr = self.samplerate
942
- n_bits = N_SYMBOLS_SAMD21 - 1
943
- eff_symbol_length = 1e3*(word_rght-word_lft)/(n_bits*sr)
944
- length_ratio = eff_symbol_length / SYMBOL_LENGTH
945
- logger.debug('symbol length_ratio (eff/supposed) %f'%length_ratio)
946
- corrected_freq = np.array(frequencies)*length_ratio
947
- logger.debug('corrected freq (using symbol length) = %s'%corrected_freq)
948
- bits = [self._get_bit_from_freq(f) for f in corrected_freq]
949
- for i, bit in enumerate(bits):
950
- if bit == None:
951
- logger.warning('cant decode frequency %i for bit at %i-%i'%(
952
- corrected_freq[i],
953
- symbols_indices[i],
954
- symbols_indices[i+1]))
955
- if None in bits:
956
- return None ########################################################
957
- bits_string = ''.join(bits)
958
- logger.debug('bits = %s'%bits_string)
959
- time_values = self._values_from_bits(bits_string)
960
- time_values['pulse at'] = (pulse_position +
961
- self.sound_extract_position -
962
- SAMD21_LATENCY*1e-6*self.samplerate)
963
- time_values['clock source'] = 'GPS' \
964
- if time_values['clock source'] == 1 else 'RTC'
965
- if self._demod_values_are_OK(time_values):
966
- return time_values
967
- else:
968
- return None
969
511
 
970
512
  class Recording:
971
513
  """
@@ -1047,7 +589,7 @@ class Recording:
1047
589
 
1048
590
  """
1049
591
 
1050
- def __init__(self, media):
592
+ def __init__(self, media, do_plots=False):
1051
593
  """
1052
594
  If multifile recording, AVfilename is sox merged audio file;
1053
595
  Set AVfilename string and check if file exists, does not read
@@ -1097,6 +639,7 @@ class Recording:
1097
639
  self.final_synced_file = None
1098
640
  self.synced_audio = None
1099
641
  self.new_rec_name = media.path.name
642
+ self.do_plots = do_plots
1100
643
  logger.debug('__init__ Recording object %s'%self.__repr__())
1101
644
  logger.debug(' in directory %s'%self.AVpath.parent)
1102
645
  recording_init_fail = ''
@@ -1119,7 +662,7 @@ class Recording:
1119
662
  elif self.get_duration() < MINIMUM_LENGTH:
1120
663
  recording_init_fail = 'file too short, %f s\n'%self.get_duration()
1121
664
  if recording_init_fail == '': # success
1122
- self.decoder = Decoder(self)
665
+ self.decoder = Decoder(self, do_plots)
1123
666
  # self._set_multi_files_siblings()
1124
667
  self._check_for_camera_error_correction()
1125
668
  else:
@@ -1171,7 +714,7 @@ class Recording:
1171
714
  """
1172
715
  if self.valid_sound:
1173
716
  val = sox.file_info.duration(_pathname(self.valid_sound))
1174
- logger.debug('duration of valid_sound %f'%val)
717
+ logger.debug('sox duration of valid_sound %f for %s'%(val,_pathname(self.valid_sound)))
1175
718
  return val #########################################################
1176
719
  else:
1177
720
  if self.probe is None:
@@ -1181,7 +724,7 @@ class Recording:
1181
724
  except:
1182
725
  logger.error('oups, cant find duration from ffprobe')
1183
726
  raise Exception('stopping here')
1184
- logger.debug('ffprobed duration is: %f sec'%probed_duration)
727
+ logger.debug('ffprobed duration is: %f sec for %s'%(probed_duration, self))
1185
728
  return probed_duration # duration in s
1186
729
 
1187
730
  def get_original_duration(self):
@@ -1238,16 +781,16 @@ class Recording:
1238
781
  end = self.get_end_time()
1239
782
  return start < datetime and datetime < end
1240
783
 
1241
- def _find_time_around(self, time, plots):
784
+ def _find_time_around(self, time):
1242
785
  """
1243
786
  Actually reads sound data and tries to decode it
1244
787
  through decoder object, if successful return a time dict, eg:
1245
- {'version': 0, 'clock source': 'GPS', 'seconds': 44, 'minutes': 57,
788
+ {'version': 0, 'seconds': 44, 'minutes': 57,
1246
789
  'hours': 19, 'day': 1, 'month': 3, 'year offset': 1,
1247
790
  'pulse at': 670451.2217 }
1248
791
  otherwise return None
1249
792
  """
1250
- if time < 0:
793
+ if time < 0: # negative = referenced from the end
1251
794
  there = self.get_duration() + time
1252
795
  else:
1253
796
  there = time
@@ -1255,7 +798,7 @@ class Recording:
1255
798
  if self.TicTacCode_channel is None:
1256
799
  return None
1257
800
  else:
1258
- return self.decoder.get_time_in_sound_extract(plots)
801
+ return self.decoder.get_time_in_sound_extract()
1259
802
 
1260
803
  def _get_timedate_from_dict(self, time_dict):
1261
804
  try:
@@ -1291,8 +834,6 @@ class Recording:
1291
834
 
1292
835
  Returns
1293
836
  -------
1294
- TYPE
1295
- DESCRIPTION.
1296
837
 
1297
838
  """
1298
839
  if t1 == None or t2 == None:
@@ -1362,7 +903,7 @@ class Recording:
1362
903
  t2 = later_recording.get_start_time()
1363
904
  return t2 - t1
1364
905
 
1365
- def get_start_time(self, plots=False):
906
+ def get_start_time(self):
1366
907
  """
1367
908
  Try to decode a TicTacCode_channel at start AND finish;
1368
909
  if successful, returns a datetime.datetime instance;
@@ -1372,7 +913,7 @@ class Recording:
1372
913
  if self.start_time is not None:
1373
914
  return self.start_time #############################################
1374
915
  cached_times = {}
1375
- def find_time(t_sec, plots=False):
916
+ def find_time(t_sec):
1376
917
  time_k = int(t_sec)
1377
918
  # if cached_times.has_key(time_k):
1378
919
  if CACHING and time_k in cached_times:
@@ -1380,7 +921,7 @@ class Recording:
1380
921
  return cached_times[time_k] ####################################
1381
922
  else:
1382
923
  logger.debug('_find_time_around() for t=%s s not cached'%time_k)
1383
- new_t = self._find_time_around(t_sec, plots)
924
+ new_t = self._find_time_around(t_sec)
1384
925
  cached_times[time_k] = new_t
1385
926
  return new_t
1386
927
  for i, pair in enumerate(TRIAL_TIMES):
@@ -1393,12 +934,12 @@ class Recording:
1393
934
  logger.warning('More than one trial: #%i/%i'%(i+1,
1394
935
  len(TRIAL_TIMES)))
1395
936
  # time_around_beginning = self._find_time_around(near_beg)
1396
- time_around_beginning = find_time(near_beg, plots)
937
+ time_around_beginning = find_time(near_beg)
1397
938
  if self.TicTacCode_channel is None:
1398
939
  return None ####################################################
1399
940
  logger.debug('Trial #%i, end at %f'%(i+1, near_end))
1400
941
  # time_around_end = self._find_time_around(near_end)
1401
- time_around_end = find_time(near_end, plots)
942
+ time_around_end = find_time(near_end)
1402
943
  logger.debug('trial result, time_around_beginning:\n %s'%
1403
944
  (time_around_beginning))
1404
945
  logger.debug('trial result, time_around_end:\n %s'%
@@ -1436,133 +977,6 @@ class Recording:
1436
977
  self.valid_sound = self.AVpath
1437
978
  return start_UTC
1438
979
 
1439
- # def _strip_TTC_and_Null(self) -> tempfile.NamedTemporaryFile:
1440
- # """
1441
- # TTC is stripped from original audio and a tempfile.NamedTemporaryFile is
1442
- # returned. If the original audio is stereo, this is simply the audio
1443
- # without the TicTacCode channel, so this fct returns a mono wav
1444
- # tempfile. But if the original audio is multitrack, two possibilities:
1445
-
1446
- # A- there's a track.txt file declaring null channels (with '0' tags)
1447
- # then those tracks are excluded too (a check is done those tracks
1448
- # have low signal and warns the user otherwise)
1449
-
1450
- # B- it's a multitrack recording but without track declaration
1451
- # (no track.txt was found in the device folder): all the tracks are
1452
- # returned into a multiwav tempfile (except TTC).
1453
-
1454
- # Notes:
1455
- # 'track.txt' is defined in device_scanner.TRACKSFN
1456
-
1457
- # Beware of track indexing: sox starts indexing tracks at 1 but code
1458
- # here uses zero based indexing.
1459
- # """
1460
- # tracks_file = self.device.folder/device_scanner.TRACKSFN
1461
- # input_file = _pathname(self.AVpath)
1462
- # # n_channels = sox.file_info.channels(input_file) # eg 2
1463
- # n_channels = self.device.n_chan
1464
- # sox_TicTacCode_channel = self.TicTacCode_channel + 1 # sox start at 1
1465
- # if n_channels == 2:
1466
- # logger.debug('stereo, so only excluding TTC (ZB idx) %i'%
1467
- # self.TicTacCode_channel)
1468
- # return self._sox_strip(input_file, [self.TicTacCode_channel]) ######
1469
- # #
1470
- # # First a check is done if the ttc tracks concur: the track detected
1471
- # # by the Decoder class, stored in Recording.TicTacCode_channel VS the
1472
- # # track declared by the user, Tracks.ttc (see device_scanner.py). If
1473
- # # not, warn the user and exit.
1474
- # trax = self.device.tracks # a Tracks dataclass instance, if any
1475
- # logger.debug('trax %s'%trax)
1476
- # if trax == None:
1477
- # return self._sox_strip(input_file, [self.TicTacCode_channel]) ######
1478
- # else:
1479
- # logger.debug('ttc channel declared for the device: %i, ttc detected: %i, non zero base indexing'%
1480
- # (trax.ttc, sox_TicTacCode_channel))
1481
- # if trax.ttc != sox_TicTacCode_channel: # warn and quit
1482
- # print('Error: TicTacCode channel detected is [gold1]%i[/gold1]'%
1483
- # sox_TicTacCode_channel, end=' ')
1484
- # print('and the file [gold1]%s[/gold1] specifies channel [gold1]%i[/gold1],'%
1485
- # (tracks_file, trax.ttc))
1486
- # print('Please correct the discrepancy and rerun. Quitting.')
1487
- # sys.exit(1)
1488
- # track_is_declared_audio = [ tag not in ['ttc','0','tc']
1489
- # for tag in trax.rawtrx]
1490
- # logger.debug('from tracks_file %s'%tracks_file)
1491
- # logger.debug('track_is_declared_audio (not ttc or 0): %s'%
1492
- # track_is_declared_audio)
1493
- # # Now find out which files are silent, ie those with sox stats "RMS lev
1494
- # # db" value inferior to DB_RMS_SILENCE_SOX (typ -50 -60 dbFS) from the
1495
- # # sox "stat" command. Ex output belwow:
1496
- # #
1497
- # # sox input.wav -n channels stats -w 0.01
1498
- # #
1499
- # # Overall Ch1 Ch2 Ch3 Ch4
1500
- # # DC offset -0.000047 -0.000047 -0.000047 -0.000016 -0.000031
1501
- # # Min level -0.060913 -0.060913 -0.048523 -0.036438 -0.002777
1502
- # # Max level 0.050201 0.050201 0.048767 0.039032 0.002838
1503
- # # Pk lev dB -24.31 -24.31 -26.24 -28.17 -50.94
1504
- # # RMS lev dB -40.33 -55.29 -53.70 -34.41 -59.75 <- this line
1505
- # # RMS Pk dB -28.39 -28.39 -30.90 -31.20 -55.79
1506
- # # RMS Tr dB -97.42 -79.66 -75.87 -97.42 -96.09
1507
- # # Crest factor - 35.41 23.61 2.05 2.76
1508
- # # Flat factor 6.93 0.00 0.00 0.97 10.63
1509
- # # Pk count 10.2 2 2 17 20
1510
- # # Bit-depth 12/16 12/16 12/16 12/16 8/16
1511
- # # Num samples 11.2M
1512
- # # Length s 232.780
1513
- # # Scale max 1.000000
1514
- # # Window s 0.010
1515
- # args = ['sox', input_file, '-n', 'channels', 'stats', '-w', '0.01']
1516
- # _, _, stat_output = sox.core.sox(args)
1517
- # logger.debug('sox stat output: \n%s'%stat_output)
1518
- # sox_RMS_lev_dB = stat_output.split('\n')[5].split()[4:]
1519
- # logger.debug('Rec %s'%self)
1520
- # logger.debug('Sox RMS %s, n_channels: %i'%(sox_RMS_lev_dB, n_channels))
1521
- # # valid audio is non silent and not ttc
1522
- # track_is_active = [float(db) > DB_RMS_SILENCE_SOX
1523
- # if idx + 1 != sox_TicTacCode_channel else False
1524
- # for idx, db in enumerate(sox_RMS_lev_dB)]
1525
- # logger.debug('track_is_active %s'%track_is_active)
1526
- # # Stored in self.device.tracks and as declared by the user, a track is
1527
- # # either:
1528
- # #
1529
- # # - an active track (because a name was given to it)
1530
- # # - the ttc track (as identified by the user)
1531
- # # - a muted track (declared by a "0" in tracks.txt)
1532
- # #
1533
- # # the following checks active tracks are effectively non silent and
1534
- # # muted tracks are effectively silent (warn the user if not but
1535
- # # proceed, giving priority to status declared in the tracks.txt file.
1536
- # # eg a non silent track will be discarded if the user tagged it with
1537
- # # a "0")
1538
- # declared_and_detected_are_same = all([a==b for a,b
1539
- # in zip(track_is_declared_audio, track_is_active)])
1540
- # logger.debug('declared_and_detected_are_same: %s'%
1541
- # declared_and_detected_are_same)
1542
- # if not declared_and_detected_are_same:
1543
- # print('Warning, the file [gold1]%s[/gold1] specifies channel usage'%
1544
- # (tracks_file))
1545
- # print('and some muted tracks are not silent (or the inverse, see below),')
1546
- # print('will proceed but if it\'s an error do necessary corrections and rerun.\n')
1547
- # table = Table(title="Tracks status")
1548
- # table.add_column("track #", justify="center", style='gold1')
1549
- # table.add_column("RMS Level", justify="center", style='gold1')
1550
- # table.add_column("Declared", justify="center", style='gold1')
1551
- # for n in range(n_channels):
1552
- # table.add_row(str(n+1), '%.0f dBFS'%float(sox_RMS_lev_dB[n]),
1553
- # trax.rawtrx[n])
1554
- # console = Console()
1555
- # console.print(table)
1556
- # if trax:
1557
- # excluded_channels = [i for i in range(n_channels)
1558
- # if not track_is_declared_audio[i]]
1559
- # else:
1560
- # excluded_channels = [self.TicTacCode_channel]
1561
- # logger.debug('excluded_channels %s (ZB idx)'%excluded_channels)
1562
- # return self._sox_strip(input_file, excluded_channels)
1563
-
1564
-
1565
-
1566
980
  def _sox_strip(self, audio_file, excluded_channels) -> tempfile.NamedTemporaryFile:
1567
981
  # building dict according to pysox.remix format.
1568
982
  # https://pysox.readthedocs.io/en/latest/api.html#sox.transform.Transformer.remix
@@ -1649,6 +1063,7 @@ class Recording:
1649
1063
  def get_samplerate(self):
1650
1064
  # return int samplerate (nominal)
1651
1065
  string = self._ffprobe_audio_stream()['sample_rate']
1066
+ logger.debug('ffprobe samplerate: %s'%string)
1652
1067
  return eval(string) # eg eval(24000/1001)
1653
1068
 
1654
1069
  def get_framerate(self):
@@ -1717,13 +1132,16 @@ class Recording:
1717
1132
 
1718
1133
  def _read_sound_find_TicTacCode(self, time_where, chunk_length):
1719
1134
  """
1720
- Loads audio data reading from self.AVpath;
1721
- Split data into channels if stereo;
1722
- Send this data to Decoder object with set_sound_extract_and_sr() to find
1135
+ If this is called for the first time for the recording, it loads audio
1136
+ data reading from self.AVpath; Split data into channels if stereo; Send
1137
+ this data to Decoder object with set_sound_extract_and_sr() to find
1723
1138
  which channel contains a TicTacCode track and sets TicTacCode_channel
1724
- accordingly: a tuple (#file, #chan). On exit, self.decoder.sound_extract
1725
- contains TicTacCode data ready to be demodulated. If not, self.TicTacCode_channel
1726
- is set to None.
1139
+ accordingly (index of channel). On exit, self.decoder.sound_extract
1140
+ contains TicTacCode data ready to be demodulated. If not,
1141
+ self.TicTacCode_channel is set to None.
1142
+
1143
+ If this has been called before (checking self.TicTacCode_channel) then
1144
+ is simply read the audio in and calls set_sound_extract_and_sr().
1727
1145
 
1728
1146
  Args:
1729
1147
  time_where : float
@@ -1735,7 +1153,8 @@ class Recording:
1735
1153
  self.decoder.set_sound_extract_and_sr()
1736
1154
 
1737
1155
  Sets:
1738
- self.TicTacCode_channel
1156
+ self.TicTacCode_channel = index of TTC chan
1157
+ self.device.ttc = index of TTC chan
1739
1158
 
1740
1159
  Returns:
1741
1160
  this Recording instance
@@ -1751,14 +1170,15 @@ class Recording:
1751
1170
  return #############################################################
1752
1171
  logger.debug('will read around %.2f sec'%time_where)
1753
1172
  dryrun = (ffmpeg
1754
- .input(str(path), ss=time_where, t=chunk_length)
1173
+ .input(str(path))
1755
1174
  .output('pipe:', format='s16le', acodec='pcm_s16le')
1756
1175
  .get_args())
1757
1176
  dryrun = ' '.join(dryrun)
1758
1177
  logger.debug('using ffmpeg-python built args to pipe wav file into numpy array:\nffmpeg %s'%dryrun)
1759
1178
  try:
1760
1179
  out, _ = (ffmpeg
1761
- .input(str(path), ss=time_where, t=chunk_length)
1180
+ # .input(str(path), ss=time_where, t=chunk_length)
1181
+ .input(str(path))
1762
1182
  .output('pipe:', format='s16le', acodec='pcm_s16le')
1763
1183
  .global_args("-loglevel", "quiet")
1764
1184
  .global_args("-nostats")
@@ -1766,28 +1186,54 @@ class Recording:
1766
1186
  .run(capture_stdout=True))
1767
1187
  data = np.frombuffer(out, np.int16)
1768
1188
  except ffmpeg.Error as e:
1769
- print('error',e.stderr)
1189
+ print('error',e.stderr)
1190
+ sound_data_var = np.std(data)
1191
+ logger.debug('extracting sound, ffmpeg output:%s with variance %f'%(data,
1192
+ sound_data_var))
1770
1193
  sound_extract_position = int(self.get_samplerate()*time_where) # from sec to samples
1771
1194
  n_chan = self.get_audio_channels_nbr()
1772
1195
  if n_chan == 1 and not self.is_video():
1773
1196
  logger.warning('file is sound mono')
1197
+ if np.isclose(sound_data_var, 0, rtol=1e-2):
1198
+ logger.warning("ffmpeg can't extract audio from %s"%self.AVpath)
1199
+ # from 1D interleaved channels to [chan1, chan2, chanN]
1774
1200
  all_channels_data = data.reshape(int(len(data)/n_chan),n_chan).T
1775
- for i_chan, chan_dat in enumerate(all_channels_data):
1776
- logger.debug('testing chan %i'%i_chan)
1201
+ if self.TicTacCode_channel == None:
1202
+ logger.debug('first call, will loop through all %i channels'%len(
1203
+ all_channels_data))
1204
+ for i_chan, chan_dat in enumerate(all_channels_data):
1205
+ logger.debug('testing chan %i'%i_chan)
1206
+ start_idx = round(time_where*self.get_samplerate())
1207
+ extract_length = round(chunk_length*self.get_samplerate())
1208
+ end_idx = start_idx + extract_length
1209
+ extract_audio_data = chan_dat[start_idx:end_idx]
1210
+ decoder.set_sound_extract_and_sr(
1211
+ extract_audio_data,
1212
+ self.get_samplerate(),
1213
+ sound_extract_position
1214
+ )
1215
+ if decoder.extract_seems_TicTacCode():
1216
+ self.TicTacCode_channel = i_chan
1217
+ self.device.ttc = i_chan
1218
+ logger.debug('found TicTacCode channel: chan #%i'%
1219
+ self.TicTacCode_channel)
1220
+ return self ################################################
1221
+ # end of loop: none found
1222
+ # self.TicTacCode_channel = None # was None already
1223
+ logger.warning('found no TicTacCode channel')
1224
+ else:
1225
+ logger.debug('been here before, TTC chan is %i'%
1226
+ self.TicTacCode_channel)
1227
+ start_idx = round(time_where*self.get_samplerate())
1228
+ extract_length = round(chunk_length*self.get_samplerate())
1229
+ end_idx = start_idx + extract_length
1230
+ chan_dat = all_channels_data[self.TicTacCode_channel]
1231
+ extract_audio_data = chan_dat[start_idx:end_idx]
1777
1232
  decoder.set_sound_extract_and_sr(
1778
- chan_dat,
1233
+ extract_audio_data,
1779
1234
  self.get_samplerate(),
1780
1235
  sound_extract_position
1781
1236
  )
1782
- if decoder.extract_seems_TicTacCode():
1783
- self.TicTacCode_channel = i_chan
1784
- self.device.ttc = i_chan
1785
- logger.debug('find TicTacCode channel, chan #%i'%
1786
- self.TicTacCode_channel)
1787
- return self ####################################################
1788
- # end of loop: none found
1789
- self.TicTacCode_channel = None
1790
- logger.warning('found no TicTacCode channel')
1791
1237
  return self
1792
1238
 
1793
1239
  def seems_to_have_TicTacCode_at_beginning(self):