tictacsync 0.98a0__py3-none-any.whl → 1.4.0b0__py3-none-any.whl

tictacsync/device_scanner.py

@@ -3,8 +3,6 @@
 # while inotifywait --recursive -e close_write . ; do python entry.py  tests/multi2/; done
 # above for linux
 
-TRACKSFN = 'tracks.txt'
-SILENT_TRACK_TOKENS = '-0n'
 
 av_file_extensions = \
 """webm mkv flv flv vob ogv ogg drc gif gifv mng avi MTS M2TS TS mov qt
@@ -18,8 +16,10 @@ M4V SVI 3GP 3G2 MXF ROQ NSV FLV F4V F4P F4A F4B 3GP AA AAC AAX ACT AIFF ALAC
 AMR APE AU AWB DSS DVF FLAC GSM IKLAX IVS M4A M4B M4P MMF MP3 MPC MSV NMF
 OGG OGA MOGG OPUS RA RM RAW RF64 SLN TTA VOC VOX WAV WMA WV WEBM 8SVX CDA MOV AVI BWF""".split()
 
+audio_ext = 'aiff wav mp3'.split()
+
 from dataclasses import dataclass
-import ffmpeg, os, sys
+import ffmpeg, os, sys, shutil
 from os import listdir
 from os.path import isfile, join, isdir
 from collections import namedtuple
@@ -28,7 +28,7 @@ from pprint import pformat
 # from collections import defaultdict
 from loguru import logger
 # import pathlib, os.path
-import sox, tempfile
+import sox, tempfile, platformdirs, filecmp
 # from functools import reduce
 from rich import print
 from itertools import groupby
@@ -36,8 +36,14 @@ from itertools import groupby
 # import distance
 try:
     from . import multi2polywav
+    from . import mamsync
+    from . import mamconf
+    from . import yaltc
 except:
     import multi2polywav
+    import mamsync
+    import mamconf
+    import yaltc
 
 MCCDIR = 'SyncedMulticamClips'
 SYNCEDFOLDER = 'SyncedMedia'
@@ -58,10 +64,9 @@ def print_grby(grby):
         print('\ngrouped by %s:'%key)
         for e in keylist:
             print(' ', e)
-
 @dataclass
 class Tracks:
-    # track numbers start at 1 for first track (as needed by sox)
+    # track numbers start at 1 for first track (as needed by sox,1 Based Index)
     ttc: int # track number of TicTacCode signal
     unused: list # of unused tracks
     stereomics: list # of stereo mics track tuples (Lchan#, Rchan#)
@@ -69,7 +74,9 @@ class Tracks:
     others: list #of all other tags: (tag, track#) tuples
     rawtrx: list # list of strings read from file
     error_msg: str # 'None' if none
-    lag_values: list # list of lag in ms, entry is None if not specified. 
+    lag_values: list # list of lags in ms, entry is None if not specified.
+    # UTC_timestamp: str # to the nearest minute ISO 8601 date and time e.g.:  "2007-04-05T14:30Z"
+
 
 @dataclass
 class Device:
@@ -210,10 +217,8 @@ class Scanner:
         top_directory : string
             String of path where to start searching for media files.
 
-        # top_dir_has_multicam : bool
-        #     If top dir is folder structures AND more than on cam
-
-        found_media_files: list of Media objects
+        found_media_files: list of dataclass Media instances encapsulating
+            the pathlibPath and the device (of Device dataclass).
     """
 
     def __init__(
@@ -244,16 +249,7 @@ class Scanner:
         CAMs = [d for d in devices if d.dev_type == 'CAM']
         return len(set(CAMs))
 
-    def get_device_in_path(self, a_media):
-        """
-        Return a device name taken from folders in path of a_media if and only
-        if all media files below it are from the same device. Goes up max two
-        levels from media; e.g., in /FOLDER2/FOLDER1/DSC00234.MOV will test up
-        to FOLDER2 as candidate.
-        """
-
-
-    def scan_media_and_build_devices_UID(self, recursive=True):
+    def scan_media_and_build_devices_UID(self, synced_root = None):
         """
         Scans Scanner.top_directory recursively for files with known audio-video
         extensions. For each file found, a device fingerprint is obtained from
@@ -269,16 +265,36 @@ class Scanner:
         Sets Scanner.input_structure = 'loose'|'ordered'
 
         """
-        files = Path(self.top_directory).rglob('*.*')
-        paths = [
-            p
-            for p in files
-            if p.suffix[1:] in av_file_extensions
-            and SYNCEDFOLDER not in p.parts # SyncedMedia
-            and MCCDIR not in p.parts # SyncedMulticamClips
-        ]
-        logger.debug('found media files %s'%paths)
-        parents = [p.parent for p in paths]
+        logger.debug(f'on entry synced_root: {synced_root}')
+        if synced_root != None: # mam mode
+            p = Path(platformdirs.user_data_dir('mamsync', 'plutz'))/mamconf.LOG_FILE
+            with open(p, 'r') as fh:
+                done = set(fh.read().split()) # sets of strings of abs path
+            logger.debug(f'done clips: {pformat(done)}')
+        files = Path(self.top_directory).rglob('*')
+        clip_paths = []
+        some_done = False
+        for raw_path in files:
+            if raw_path.suffix[1:] in av_file_extensions:
+                if SYNCEDFOLDER not in raw_path.parts: # SyncedMedia
+                    if MCCDIR not in raw_path.parts: # SyncedMulticamClips
+                        if '_ISO' not in [part[-4:] for part in raw_path.parts]: # exclude ISO wav files
+                            if synced_root != None and str(raw_path) in done:
+                                logger.debug(f'{raw_path} done')
+                                some_done = True
+                                continue
+                            else:
+                                clip_paths.append(raw_path)
+        if some_done:
+            print('Somme media files were already synced...')
+        logger.debug('found media files %s'%clip_paths)
+        if len(clip_paths) == 0:
+            print('No media found, bye.')
+            sys.exit(0)
+            # self.found_media_files = []
+            # self.input_structure = 'loose'
+            # return
+        parents = [p.parent for p in clip_paths]
         logger.debug('found parents %s'%pformat(parents))
         # True if all elements are identical
         AV_files_have_same_parent = parents.count(parents[0]) == len(parents)
@@ -292,7 +308,7 @@ class Scanner:
             # check later if inside each folder, media have same device
             # for now, we'll guess structure is 'ordered'
             self.input_structure = 'ordered'
-        for p in paths:
+        for p in clip_paths:
             new_media = media_at_path(self.input_structure, p) # dev UID set here
             self.found_media_files.append(new_media)
         # for non UIDed try building UID from filenam
@@ -334,17 +350,19 @@ class Scanner:
         logger.debug('Scanner.found_media_files = %s'%pformat(self.found_media_files))
         if self.input_structure == 'ordered':
             self._confirm_folders_have_same_device()
-            # self._use_folder_as_device_name()
-            devices = set([m.device for m in self.found_media_files])
-            audio_devices = [d for d in devices if d.dev_type == 'REC']
-            for recorder in audio_devices:
-                # process tracks.txt for audio recorders
-                recorder.tracks = self._get_tracks_from_file(recorder)
-                # logging only:
-                if recorder.tracks:
-                    if not all([lv == None for lv in recorder.tracks.lag_values]):
-                        logger.debug('%s has lag_values %s'%(
-                                recorder.name, recorder.tracks.lag_values))
+            # [TODO]  move this where Recordings have been TCed for any tracks timestamps
+            # <begin>
+            # devices = set([m.device for m in self.found_media_files])
+            # audio_devices = [d for d in devices if d.dev_type == 'REC']
+            # for recorder in audio_devices:
+            #     # process tracks.txt for audio recorders
+            #     recorder.tracks = self._get_tracks_from_file(recorder)
+            #     # logging only:
+            #     if recorder.tracks:
+            #         if not all([lv == None for lv in recorder.tracks.lag_values]):
+            #             logger.debug('%s has lag_values %s'%(
+            #                     recorder.name, recorder.tracks.lag_values))
+            # </end>
             # check if device is in fact two parents up (and parent = ROLLxx):
             # Group media by folder 2up and verify all media for each
             # group have same device.
@@ -371,11 +389,10 @@ class Scanner:
                 for m in self.found_media_files:
                     if m.device.UID == UID:
                         m.device.name = name
-        # logger.debug('renamed device media: %s'%pformat(self.found_media_files))
         no_name_devices = [m.device for m in self.found_media_files
                                             if not m.device.name]
                             # possible if self.input_structure == 'loose'
-        def _try_name_from_metadata(media):
+        def _try_name_from_metadata(media): # unused for now
             # search model and make from fprobe
             file = Path(media.path)
             logger.debug('trying to find maker model for %s'%file)
@@ -391,7 +408,6 @@ class Scanner:
             # could reside in ['format','tags','com.apple.quicktime.model'],
             # or ['format','tags','model'],
             # or ['streams'][0]['tags']['vendor_id'])  :-(
-
         for anon_dev in no_name_devices:
             medias = self.get_media_for_device(anon_dev)
             guess_name = _try_name_from_files(medias)
@@ -405,50 +421,6 @@ class Scanner:
         if not any(dev_is_REC): # no audio recordings!
             print('\rNo audio recording found, nothing to sync, bye.')
             sys.exit(0)
-        # print('devices 312 %s'%set([m.device for m in self.found_media_files]))
-
-    def _get_tracks_from_file(self, device) -> Tracks:
-        """        
-        Look for eventual track names in TRACKSFN file, stored inside the
-        recorder folder alongside the audio files. If there, returns a Tracks
-        object, if not returns None. 
-        """        
-        source_audio_folder = device.folder
-        tracks_file = source_audio_folder/TRACKSFN
-        track_names = False
-        a_recording = [m for m in self.found_media_files
-                                                if m.device == device][0]
-        logger.debug('a_recording for device %s : %s'%(device, a_recording))
-        nchan = sox.file_info.channels(str(a_recording.path))
-        if os.path.isfile(tracks_file):
-            logger.debug('found file: %s'%(TRACKSFN))
-            tracks = self._parse_track_values(tracks_file)
-            if tracks.error_msg:
-                print('\nError parsing [gold1]%s[/gold1] file: %s, quitting.\n'%
-                    (tracks_file, tracks.error_msg))
-                sys.exit(1)
-            logger.debug('parsed tracks %s'%tracks)
-            ntracks = 2*len(tracks.stereomics)
-            ntracks += len(tracks.mix)
-            ntracks += len(tracks.unused)
-            ntracks += len(tracks.others)
-            ntracks += 1 # for ttc track
-            logger.debug(' n chan: %i n tracks file: %i'%(nchan, ntracks))
-            if ntracks != nchan:
-                print('\nError parsing %s content'%tracks_file)
-                print('incoherent number of tracks, %i vs %i quitting\n'%
-                                                    (nchan, ntracks))
-                sys.exit(1)
-            err_msg = tracks.error_msg
-            if  err_msg != None:
-                print('\nError, quitting: in file %s, %s'%(tracks_file, err_msg))
-                raise Exception
-            else:
-                logger.debug('tracks object%s'%tracks)
-                return tracks
-        else:
-            logger.debug('no tracks.txt file found')
-            return None
 
     def _confirm_folders_have_same_device(self):
         """
@@ -586,184 +558,6 @@ class Scanner:
         logger.debug('n_CAM_folder %i'%n_CAM_folder)
         return
 
-    def _parse_track_values(self, tracks_file) -> Tracks:
-        """
-        read track names for naming separated ISOs
-        from tracks_file.
-
-        tokens looked for: mix; mix L; mix R; 0 and TC
-
-        repeating "mic*" pattern signals a stereo track
-        and entries will correspondingly panned into
-        a stero mix named mixL.wav and mixL.wav
-
-        mic L # spaces are ignored |
-        mic R                      | stereo pair
-        micB L
-        micB R
-
-        Returns: a Tracks instance:
-                # track numbers start at 1 for first track (as needed by sox)
-                ttc: int # track number of TicTacCode signal
-                unused: list # of unused tracks
-                stereomics: list # of stereo mics track tuples (Lchan#, Rchan#)
-                mix: list # of mixed tracks, if a pair, order is L than R
-                others: list #of all other tags: (tag, track#) tuples
-                rawtrx: list # list of strings read from file
-                error_msg: str # 'None' if none
-        e.g.: Tracks(   ttc=2,
-                        unused=[],
-                        stereomics=[('mic', (4, 3)), ('mic2', (6, 5))],
-                        mix=[], others=[('clics', 1)],
-                        rawtrx=['clics', 'TC', 'micL', 'micR', 'mic2L;1000', 'mic2R;1000', 'mixL', 'mixR'],
-                        error_msg=None, lag_values=[None, None, None, None, '1000', '1000', None, None])
-        """
-        def _WOspace(chaine):
-            ch = [c for c in chaine if c != ' ']
-            return ''.join(ch)
-        # def _WO_LR(chaine):
-        #     ch = [c for c in chaine if c not in 'LR']
-        #     return ''.join(ch)
-        # def _seemsStereoMic(tag):
-        #     # is tag likely a stereo pair tag?
-        #     # should starts with 'mic' and ends with 'l' or 'r'
-        #     return tag[1:4]=='mic' and tag[0] in 'lr'
-        file=open(tracks_file,"r")
-        whole_txt = file.read()
-        logger.debug('file %s all_lines:\n%s'%(tracks_file, whole_txt))
-        tracks_lines_wspaces = [l.split('#')[0] for l in whole_txt.splitlines()
-                                        if len(l) > 0 ]
-        tracks_lines = [_WOspace(l) for l in tracks_lines_wspaces if len(l) > 0 ]
-        rawtrx = [l for l in tracks_lines_wspaces if len(l) > 0 ]
-        # add index with tuples, starting at 1
-        logger.debug('tracks_lines whole: %s'%tracks_lines)
-        def _detach_lag_value(line):
-            # look for ";number" ending any line, returns a two-list
-            splt = line.split(';')
-            if len(splt) == 1:
-                splt += [None]
-                if len(splt) != 2:
-                    # error
-                    print('\nText error in %s, line %s has too many ";"'%(
-                            tracks_file, line))
-            return splt
-        tracks_lines, lag_values = zip(*[_detach_lag_value(l) for l 
-                                                    in tracks_lines])
-        lag_values = [e for e in lag_values] # from tuple to list
-        # logger.debug('tracks_lines WO lag: %s'%tracks_lines)
-        tracks_lines = [l.lower() for l in tracks_lines]
-        logger.debug('tracks_lines lower case: %s'%tracks_lines)
-        # print(lag_values)
-        logger.debug('lag_values: %s'%lag_values)
-        tagsWOl_r = [e[:-1] for e in tracks_lines] # skip last letter
-        logger.debug('tags WO LR letter %s'%tagsWOl_r)
-        # find idx of start of pairs
-        # ['clics', 'TC', 'micL', 'micR', 'mic2L', 'mic2R', 'mixL', 'mixR']
-        def _micOrmix(a,b):
-            # test if same and mic mic or mix mix
-            if len(a) == 0:
-                return False
-            return (a == b) and (a in 'micmix')
-        pair_idx_start =[i for i, same in enumerate([_micOrmix(a,b) for a,b
-                        in zip(tagsWOl_r,tagsWOl_r[1:])]) if same]
-        logger.debug('pair_idx_start %s'%pair_idx_start)
-        def LR_OK(idx):
-            # in tracks_lines, check if idx ends a LR pair
-            # delays, if any, have been removed
-            a = tracks_lines[idx][-1]
-            b = tracks_lines[idx+1][-1]
-            return a+b in ['lr', 'rl']
-        LR_OKs = [LR_OK(p) for p in pair_idx_start]
-        logger.debug('LR_OKs %s'%LR_OKs)
-        if not all(LR_OKs):
-            print('\nError in %s'%tracks_file)
-            print('Some tracks are paired but not L and R: %s'%rawtrx)
-            print('quitting...')
-            quit()
-        complete_pairs_idx = pair_idx_start + [i + 1 for i in pair_idx_start]
-        singles = set(range(len(tracks_lines))).difference(complete_pairs_idx)
-        logger.debug('complete_pairs_idx %s'%complete_pairs_idx)
-        logger.debug('singles %s'%singles)
-        singles_tag = [tracks_lines[i] for i in singles]
-        logger.debug('singles_tag %s'%singles_tag)
-        n_tc_token = sum([t == 'tc' for t in singles_tag])
-        logger.debug('n tc tags %s'%n_tc_token)
-        if n_tc_token == 0:
-            print('\nError in %s'%tracks_file)
-            print('with %s'%rawtrx)
-            print('no TC track found, quitting...')
-            quit()
-        if n_tc_token > 1:
-            print('\nError in %s'%tracks_file)
-            print('with %s'%rawtrx)
-            print('more than one TC track, quitting...')
-            quit()
-        output_tracks = Tracks(None,[],[],[],[],rawtrx,None,[])
-        output_tracks.ttc = tracks_lines.index('tc') + 1 # 1st = 1
-        logger.debug('ttc_chan %s'%output_tracks.ttc)
-        zeroed = [i+1 for i, t in enumerate(tracks_lines) if t == '0']
-        logger.debug('zeroed %s'%zeroed)
-        output_tracks.unused = zeroed
-        output_tracks.others = [(st, tracks_lines.index(st)+1) for st
-                                in singles_tag if st not
-                                in ['tc', 'monomix', '0']]
-        logger.debug('output_tracks.others %s'%output_tracks.others)
-        # check for monomix
-        if 'monomix' in tracks_lines:
-            output_tracks.mix = [tracks_lines.index('monomix')+1]
-        else:
-            output_tracks.mix = []
-        # check for stereo mix
-        def _findLR(i_first):
-            # returns L R indexes (+1 for sox non zero based indexing)
-            i_2nd = i_first + 1
-            a = tracks_lines[i_first][-1] # l|r at end
-            b = tracks_lines[i_2nd][-1] # l|r at end
-            if a == 'l':
-                if b == 'r':
-                    # sequence is mixL mixR
-                    return i_first+1, i_2nd+1
-                else:
-                    print('\nError in %s'%tracks_file)
-                    print('with %s'%rawtrx)
-                    print('can not find stereo mix')
-                    quit()
-            elif a == 'r':
-                if b == 'l':
-                    # sequence is mixR mixL
-                    return i_2nd+1, i_first+1
-                else:
-                    print('\nError in %s'%tracks_file)
-                    print('with %s'%rawtrx)
-                    print('can not find stereo mix')
-                    quit()
-        logger.debug('for now, output_tracks.mix %s'%output_tracks.mix)
-        mix_pair = [p for p in pair_idx_start if tracks_lines[p][1:] == 'mix']
-        if len(mix_pair) == 1:
-            # one stereo mix, remove it from other pairs
-            i = mix_pair[0]
-            LR_pair =  _findLR(i)
-            logger.debug('LR_pair %s'%str(LR_pair))
-            pair_idx_start.remove(i)
-            # consistency check
-            if output_tracks.mix != []:
-                # already found a mono mix above!
-                print('\nError in %s'%tracks_file)
-                print('with %s'%rawtrx)
-                print('found a mono mix AND a stereo mix')
-                quit()
-            output_tracks.mix = LR_pair
-        logger.debug('finally, output_tracks.mix %s'%str(output_tracks.mix))
-        logger.debug('remaining pairs %s'%pair_idx_start)
-        # those are stereo pairs
-        stereo_pairs = []
-        for first_in_pair in pair_idx_start:
-            suffix = tracks_lines[first_in_pair][:-1]
-            stereo_pairs.append((suffix, _findLR(first_in_pair)))
-        logger.debug('stereo_pairs %s'%stereo_pairs)
-        output_tracks.stereomics = stereo_pairs
-        logger.debug('finished: %s'%output_tracks)
-        return output_tracks