labdata 0.0.3__py3-none-any.whl

labdata/copy.py

@@ -0,0 +1,351 @@
+from .utils import *
+from warnings import warn
+
+def _copyfile_to_upload_server(filepath, local_path=None, server_path = None,overwrite = False):
+    '''
+
+    This is an internal function, not meant to be called by the end user, call copy_to_upload_server instead.
+
+    This is a support function that will copy data between computers; it will not overwrite, unless forced.
+    
+    It will raise an exception if the files are already there unless overwrite is true.
+    
+    Does not insert to the Upload table.
+
+
+    Parameters
+    ----------
+    filepath : str or Path
+        Path to the file to copy, relative to local_path
+    local_path : str or Path, optional
+        Source directory containing the file. If None, uses first path from preferences
+    server_path : str or Path, optional
+        Destination directory to copy to. If None, uses upload_path from preferences
+    overwrite : bool, default False
+        Whether to overwrite existing files at the destination
+
+    Returns
+    -------
+    dict
+        Dictionary containing:
+        - src_path: Relative path to the file
+        - src_md5: MD5 hash of the file
+        - src_size: Size of the file in bytes 
+        - src_datetime: Creation timestamp of the file
+
+    Raises
+    ------
+    OSError
+        If filepath is a directory
+        If destination file exists and overwrite=False
+        If copy operation fails
+
+
+    Joao Couto - labdata 2024
+    
+    '''
+    
+    src = Path(local_path)/filepath
+    if src.is_dir():
+        raise OSError(f'Can only handle files {src}; copy each file in the folder separately.')
+    dst = Path(server_path)/filepath
+    if not overwrite and dst.exists() and not src == dst:
+        raise OSError(f'File {dst} exists; will not overwrite.')
+    hash = compute_md5_hash(src)  # computes the hash
+    srcstat = src.stat()
+    file_size = srcstat.st_size
+    from shutil import copy2
+    dst.parent.mkdir(parents=True, exist_ok = True)
+    if not src == dst: 
+        try:
+            copy2(src, dst)
+        except:
+            raise OSError(f'Could not copy {src} to {dst}.')
+    else:
+        # if the source and destination are the same, don't copy - just return cause its in the same server..
+        print('[copy_to_upload_server] Source and destination are the same, not copying.', flush = True)
+    filepath = str(filepath).replace(pathlib.os.sep,'/').replace('//','/')
+    return dict(src_path = filepath,
+                src_md5 = hash,
+                src_size = file_size,
+                src_datetime = datetime.fromtimestamp(srcstat.st_ctime))
+
+def copy_to_upload_server(filepaths, local_path = None, server_path = None,
+                          upload_storage = None,
+                          overwrite = False, 
+                          n_jobs = 8,
+                          job_rule = None,
+                          parse_filename = True,
+                          **kwargs):
+    '''
+    Copy data between computers; it will not overwrite, unless forced.
+
+    Returns a list of dictionaries with the file paths and md5 checksums.
+
+    Inserts in the Upload table. 
+    Parameters
+    ----------
+    filepaths : list
+        List of file paths to copy to upload server
+    local_path : str, optional
+        Local path where files are stored. If None, uses first path from preferences
+    server_path : str, optional
+        Path on upload server. If None, uses upload_path from preferences
+    upload_storage : str, optional
+        Storage location for uploaded files. If None, uses upload_storage from preferences
+    overwrite : bool, default False
+        Whether to overwrite existing files on server
+    n_jobs : int, default 8
+        Number of parallel jobs for copying files
+    job_rule : str, optional
+        Rule to apply during upload
+    parse_filename : bool, default True
+        Whether to parse metadata from filenames based on path rules
+    **kwargs : dict
+        Additional metadata to store with upload (e.g. subject_name, session_name)
+
+    Returns
+    -------
+    dict
+        Dictionary containing upload job information
+
+    Joao Couto - labdata 2024
+    '''
+    
+    from .schema import UploadJob,Setup,Subject,Session,Dataset,dj
+    if local_path is None:  # get the local_path from the preferences
+        if not 'local_paths' in prefs.keys():
+            raise OSError('Local data path [local_paths] not specified in the preference file.')
+        local_path = prefs['local_paths'][0]
+    if server_path is None: # get the upload_path from the preferences
+        if not 'upload_path' in prefs.keys():
+            raise OSError('Upload storage [upload_path], not specified in the  preference file.')    
+        server_path = prefs['upload_path']
+    if upload_storage is None: # get the upload_storage from the preferences, where data will be stored.
+        if not 'upload_storage' in prefs.keys():
+            raise OSError('Upload storage [upload_storage], not specified in the  preference file.')    
+        upload_storage = prefs['upload_storage']
+        
+    if not type(filepaths) is list: # Check if the filepaths are in a list
+        raise ValueError('Input filepaths must be a list of paths.')
+    # replace local_path if the user passed it like that by accident.
+    filepaths = [str(f).replace(str(local_path),'') for f in filepaths]
+    # remove trailing / or \
+    filepaths = [f if not f.startswith(pathlib.os.sep) else f[1:] for f in filepaths]
+    # make unique
+    filepaths = [f for f in np.unique(filepaths)]
+    
+    if any_path_uploaded(filepaths):
+        if not job_rule in ['replace']:
+            raise OSError('Path was already uploaded {0}'.format(Path(filepaths[0]).parent))
+        else:
+            warn('[copy_to_upload_server] Files exist in the server and will be replaced.')
+    
+    if parse_filename: # parse filename based on the path rules
+        tmp = parse_filepath_parts(filepaths[0])
+        for k in tmp.keys():
+            if not k in kwargs.keys():
+                kwargs[k] = tmp[k]
+
+    n_jobs = validate_num_jobs_joblib(n_jobs)   # avoid nested parallelism.
+    # copy and compute checksum for all paths in parallel.
+    res = Parallel(n_jobs = n_jobs)(
+        delayed(_copyfile_to_upload_server)(path,
+                                            local_path = local_path,
+                                            server_path = server_path,
+                                            overwrite = overwrite) for path in filepaths)
+    # Add it to the upload table
+    # check the job id
+    with dj.conn().transaction:
+        #print(kwargs)
+        if "setup_name" in kwargs.keys():
+            Setup.insert1(kwargs, skip_duplicates = True, ignore_extra_fields = True) # try to insert setup
+        if "dataset_name" in kwargs.keys() and "session_name" in kwargs.keys() and "subject_name" in kwargs.keys():
+            if not len(Subject() & dict(subject_name=kwargs['subject_name'])):
+                try:
+                    Subject.insert1(kwargs, skip_duplicates = True,ignore_extra_fields = True) # try to insert subject
+                except Exception as err:
+                    warn(f'Could not insert subject {kwargs}')
+                    print(err, flush = True)
+                # needs date of birth and sex
+            if not len(Session() & dict(subject_name=kwargs['subject_name'],
+                                        session_name = kwargs['session_name'])):
+                Session.insert1(kwargs, skip_duplicates = True,ignore_extra_fields = True) # try to insert session
+            if not len(Dataset() & dict(subject_name = kwargs['subject_name'],
+                                        session_name = kwargs['session_name'],
+                                        dataset_name = kwargs['dataset_name'])):
+                Dataset.insert1(kwargs, skip_duplicates = True, ignore_extra_fields = True) # try to insert dataset
+         # this is a brute force way... there should be a better way of doing this but the auto-increment not return in datajoint...
+        attempts = 10
+        jobid = UploadJob().fetch('job_id')
+        if len(jobid):
+            jobid = np.max(jobid) + 1 
+        else:
+            jobid = 1
+        job_insert_failed = 1
+        for iattempt in range(attempts):
+            try: 
+                UploadJob.insert1(dict(job_id = jobid, 
+                                       job_status = "ON SERVER",
+                                       upload_storage = upload_storage,
+                                       job_rule = job_rule,
+                                       **kwargs),
+                                ignore_extra_fields = True) # Need to insert the dataset first if not there
+                job_insert_failed = 0
+                break # we have the job id
+            except Exception as err:
+                jobid += 1
+                print(err) # we don't have it, do it again
+        if job_insert_failed:
+            raise ValueError(f'Job insert failed because could not add {jobid} to the UploadJob queue.')
+        res = [dict(r, job_id = jobid) for r in res] # add dataset through kwargs
+        UploadJob.AssignedFiles.insert(res, ignore_extra_fields=True)
+        # the upload server will run the checksum and upload the files.
+    return res
+
+def any_path_uploaded(filepaths):
+    '''
+    any_path_uploaded(filepaths)
+
+    Checks if any of the provided file paths:
+    - Exist in the File table (already uploaded)
+    - Exist in the UploadJob.AssignedFiles table (queued for upload)
+    - Exist in the ProcessedFile table (original file deleted because the dataset was processed with a Rule)
+
+    Parameters
+    ----------
+    filepaths : list
+        List of file paths to check
+
+    Returns
+    -------
+    bool
+        True if any of the files are already uploaded or in the upload queue
+        False if none of the files are uploaded or queued
+    '''
+    from .schema import UploadJob, File, ProcessedFile
+    # check if the paths are in "Upload or in Files"
+    paths = []
+    src_paths = []
+    for p in filepaths: # if true for any of the filepaths return True
+        pn = p.replace(pathlib.os.sep,'/').replace('//','/')
+        paths.append(dict(file_path = pn))
+        src_paths.append(dict(src_path = pn))
+    if len((File() & paths)) > 0:
+        warn(f'Found paths in File table {File() & paths}')
+        return True
+    if len(UploadJob.AssignedFiles() & src_paths) > 0:
+        warn(f'Found paths in UploadJob table {UploadJob.AssignedFiles() & src_paths}')
+        return True
+    if len(ProcessedFile() & paths) > 0:
+        warn(f'Found paths in ProcessedFile table {ProcessedFile() & paths}')
+        return True
+    return False # Otherwise return False
+
+def all_paths_uploaded(filepaths):
+    '''
+    all_paths_uploaded(filepaths)
+
+    Checks if all of the provided file paths exist in the File table (already uploaded)
+    
+    This function only checks the File table, unlike any_path_uploaded() which also checks
+    UploadJob.AssignedFiles and ProcessedFile tables.
+
+    Parameters
+    ----------
+    filepaths : list
+        List of file paths to check
+
+    Returns
+    -------
+    bool
+        True if all of the files are already uploaded
+        False if any files are missing from the File table
+    '''
+    from .schema import UploadJob, File, ProcessedFile
+    # check if the paths are in "Upload or in Files"
+    paths = []
+    src_paths = []
+    for p in filepaths: # if true for any of the filepaths return True
+        pn = p.replace(pathlib.os.sep,'/').replace('//','/')
+        paths.append(dict(file_path = pn))
+        src_paths.append(dict(src_path = pn))
+    if len((File() & paths)) == len(filepaths):
+        return True
+    
+    warn(f'Could not find some files in the table {File() & paths} when trying to find {filepaths}') 
+    return False # Otherwise return False
+                    
+def clean_local_path(local_path = None, filterkeys = [], dry_run = False):
+    ''' 
+    clean_local_path(local_path = None, filterkeys = [], dry_run = False)
+
+    Remove local files that are already in the database from a local path
+    Clean local files that have already been uploaded to the database.
+    
+    This function checks files in a local directory against the database and removes
+    local copies that have already been successfully uploaded (verified by MD5 checksum).
+    
+    Files are only deleted if they exist in either the File or ProcessedFile tables
+      AND their MD5 checksums match exactly
+    
+    Files with mismatched checksums are kept and reported
+    
+    Empty directories are not automatically removed (TODO)
+
+    Parameters
+    ----------
+    local_path : str or Path, optional
+        Path to local directory to clean. If None, uses first path from preferences.
+    filterkeys : list, optional
+        List of strings to filter filenames by. Only files containing all filterkeys will be checked.
+    dry_run : bool, optional
+        If True, only prints what would be deleted without actually deleting files.
+        
+    Returns
+    -------
+    deleted : list
+        List of files that were deleted (or would be deleted in dry_run mode)
+    keep : list 
+        List of files that were kept because their checksums did not match the database
+        
+    Joao Couto - labdata 2024
+    '''
+    from .schema import File, ProcessedFile
+    if local_path is None:
+        local_path = prefs['local_paths'][0]
+    filelist = Path(local_path).rglob('*')
+    local_filelist = list(filter(lambda x: x.is_file(),filelist))
+    remote_filelist = np.array(drop_local_path(local_filelist))
+    # check that the files follow a filter
+    selection = np.ones_like(remote_filelist, dtype=bool)
+    for f in filterkeys:
+        selection *= np.array([1 if f in str(k) else 0 for k in remote_filelist],dtype=bool)
+    remote_keys_all = [dict(file_path = str(r).replace('\\','/')) for r in remote_filelist[selection]]
+    # get the remote keys
+    remote_keys = (File() & remote_keys_all).fetch(as_dict = True)
+    remote_keys += (ProcessedFile() & remote_keys_all).fetch(as_dict = True)
+    keys = [dict(r,
+                 local_path = Path(local_path)/r['file_path']) for r in remote_keys]
+    res,comparison = compare_md5s([r['local_path'] for r in keys],[r['file_md5'] for r in keys],
+                                  full_output = True,
+                                  show_progress = True,
+                                  suppress_file_not_found = True) # so if the files disappear it doesn't crash.
+    if not res:
+        print(tcolor['r'](f'{np.sum(~np.array(comparison))} local files have different checksums.'),flush = True)
+        for k,r in zip(keys,comparison):
+            if not r:
+                print(tcolor['r'](f" \t {k['local_path']}"),flush = True)
+    keep  = []
+    deleted = []
+    for k,r in zip(keys,comparison):
+        if not r:
+            keep.append(k)
+        else:
+            if not dry_run:
+                os.unlink(k['local_path'])
+            deleted.append(k['local_path'])
+    # check if the "deleted" folders are empty
+    for f in deleted:
+        print(f.parent)
+    return deleted,keep

labdata/rules/__init__.py

@@ -0,0 +1,78 @@
+'''Upload rules for lab data management.
+
+This module provides rules for processing data uploads to the database.
+Rules handle validation, preprocessing, and storage of different data types.
+
+Default rules included:
+- UploadRule: Base rule for generic file uploads
+- EphysRule: Rule for electrophysiology data (SpikeGLX)
+- TwoPhotonRule: Rule for two-photon microscopy data (ScanImage/Scanbox)
+- OnePhotonRule: Rule for one-photon imaging data (Widefield - labcams)
+- MiniscopeRule: Rule for (UCLA) Miniscope imaging data
+- FixedBrainRule: Rule for fixed tissue microscopy data
+- ReplaceRule: Rule for replacing existing files
+
+Custom rules can be added to the user_preferences.json configuration.
+'''
+
+from .utils import *
+from .ephys import *
+from .imaging import *
+
+rulesmap = dict(none = UploadRule,
+                ephys = EphysRule,
+                two_photon = TwoPhotonRule,
+                one_photon = OnePhotonRule,
+                fixed_brain = FixedBrainRule,
+                miniscope = MiniscopeRule,
+                replace = ReplaceRule)
+
+if 'upload_rules' in prefs.keys():
+    for k in prefs['upload_rules']:
+        # TODO: this may need a special field for imports?
+        rulesmap[k] = eval(prefs['upload_rules'][k])
+
+def process_upload_jobs(key = None,
+                        rule = 'all',
+                        n_jobs = 8,
+                        job_host = None,
+                        force = False,
+                        prefs = None):
+    '''
+    Process UploadJobs using UploadRule(s).
+    Custom upload rules can be added as plugins, just include in prefs['upload_rules']
+
+    Joao Couto - labdata 2024
+    '''
+    from tqdm import tqdm
+    from ..schema import UploadJob
+    def _job(j,rule = None, prefs = None):
+        jb = (UploadJob() & f'job_id = {j}').fetch()
+        if len(jb):
+            if jb[0]['job_rule'] in rulesmap.keys(): 
+                rl = rulesmap[jb[0]['job_rule']](j, prefs = prefs)
+            else:
+                rl = UploadRule(j,prefs = prefs)
+            res = rl.apply()
+        return res
+    if key is None:
+        jobs = (UploadJob() & 'job_waiting = 1').fetch('job_id')
+        if not job_host is None:
+            if job_host == 'self':
+                job_host = prefs['hostname']
+            jobs = (UploadJob() &
+                    'job_waiting = 1' &
+                    f'job_host = "{job_host}"').fetch('job_id')
+    else:
+        jobs = (UploadJob() & key).fetch('job_id')
+    if force:
+        for j in jobs:
+            UploadJob().update1(dict(job_id  = j, job_waiting = 1)) # reset job
+    if len(jobs) == 1:
+        res = [_job(jobs[0], rule = rule, prefs = prefs)]
+    else:   
+        res = Parallel(backend='loky',n_jobs = n_jobs)(delayed(_job)(u,
+                                                                     rule = rule,
+                                                                     prefs = prefs) 
+                       for u in tqdm(jobs,desc = "Running upload jobs: "))
+    return res

labdata/rules/ephys.py

@@ -0,0 +1,188 @@
+from .utils import *
+
+class EphysRule(UploadRule):
+    def __init__(self, job_id,prefs = None):
+        super(EphysRule,self).__init__(job_id = job_id,prefs = prefs)
+        self.rule_name = 'ephys'
+        self.max_concurrent = 2
+        
+    def _apply_rule(self):
+        # compress ap files or lf files.
+        files_to_compress = list(filter(lambda x: ('.ap.bin' in x) or
+                                        ('.lf.bin' in x),
+                                        self.src_paths.src_path.values))
+        n_jobs = DEFAULT_N_JOBS
+        # compress these in parallel, will work for multiprobe sessions faster?
+        if len(files_to_compress): # in some cases data might have already been compressed
+            self.set_job_status(job_status = 'WORKING',
+                                job_log = datetime.now().strftime(f'Compressing {len(files_to_compress)} files [%Y %m %d %H:%M:%S]'))
+            res = Parallel(n_jobs = n_jobs)(delayed(compress_ephys_file)(
+                filename,
+                local_path = self.local_path,
+                n_jobs = n_jobs,
+                prefs = self.prefs) for filename in files_to_compress)
+            new_files = np.stack(res).flatten() # stack the resulting files and add them to the path
+            self.set_job_status(job_status = 'WORKING',
+                                job_log = datetime.now().strftime(f'Compressed {len(files_to_compress)} files [%Y %m %d %H:%M:%S]'))
+            self._handle_processed_and_src_paths(files_to_compress, new_files)
+        return self.src_paths
+    
+    def _post_upload(self):
+        if not self.dataset_key is None:
+            from ..schema import EphysRecording 
+            EphysRecording().add_spikeglx_recording(self.dataset_key)
+            EphysRecording().add_nidq_events(self.dataset_key)
+                
+############################################################################################################
+############################################################################################################
+
+        
+def compress_ephys_file(filename, local_path = None, 
+                        ext = '.bin',
+                        n_jobs = DEFAULT_N_JOBS,
+                        check_after_compress = True,
+                        prefs = None):
+    '''
+    Compress ephys data
+    '''
+    if prefs is None:
+        from ..utils import prefs
+    if local_path is None:
+        local_path = prefs['local_paths'][0]
+    local_path = Path(local_path)
+    
+    from spks.spikeglx_utils import read_spikeglx_meta
+    
+    binfile = local_path/filename
+    if not binfile.exists():
+        raise OSError(f'Could not find binfile to compress ephys {binfile}')
+    
+    metafile = local_path/str(filename).replace(ext,'.meta')    
+    if not metafile.exists():
+        raise OSError(f'Could not find metafile to compress ephys {metafile}')
+
+    meta = read_spikeglx_meta(metafile)  # to get the sampling rate and nchannels
+    srate = meta['sRateHz']
+    nchannels = meta['nSavedChans']
+    from mtscomp import compress, decompress
+    # Compress a .bin file into a pair .cbin (compressed binary file) and .ch (JSON file).
+    cbin,ch = (str(binfile).replace(ext,'.cbin'),str(binfile).replace(ext,'.ch'))
+    compress(binfile, cbin, ch,
+             sample_rate = srate, n_channels = int(nchannels),
+             check_after_compress = check_after_compress,
+             chunk_duration = 1, dtype=np.int16, n_threads = n_jobs)
+    return cbin.replace(str(local_path),'').strip(pathlib.os.sep),ch.replace(str(local_path),'').strip(pathlib.os.sep)
+    
+
+def get_probe_configuration(meta):
+    '''
+    Meta can be a file or a dictionary.
+    Uses spks for now to parse the metadata.
+    '''
+    if not hasattr(meta,'keys'):
+        meta = Path(meta)
+        if not meta.exists():
+            raise OSError(f'File not found: {meta}')
+        try:
+            from spks.spikeglx_utils import read_spikeglx_meta
+            # TODO: consider porting a minimal version over
+        except:
+            raise OSError('Could not import spks: install from https://github.com/spkware/spks')
+        meta = read_spikeglx_meta(meta)
+    
+    probe_type = str(int(meta['imDatPrb_type']))
+    recording_software = 'spikeglx' # make this work with openephys also
+    return dict(probe_id = str(int(meta['imDatPrb_sn'])),
+                recording_software = recording_software,
+                recording_duration = meta['fileTimeSecs'],
+                sampling_rate = meta['sRateHz'],
+                probe_type = probe_type,
+                probe_n_shanks = 4 if probe_type in ['24','2013','2014'] else 1,
+                probe_gain = meta['conversion_factor_microV'][0],
+                probe_n_channels = len(meta['channel_idx']),
+                channel_idx = meta['channel_idx'],
+                channel_coords = meta['coords'],
+                channel_shank = meta['channel_shank'],
+                probe_recording_channels = int(meta['nSavedChans']-1))
+
+
+def ephys_noise_statistics_from_file(filepath,channel_indices, gain, sampling_rate = 30000, duration = 60):
+    '''
+    statistics = ephys_noise_statistics_from_file(filepath,channel_indices, gain, sampling_rate = 30000, duration = 60)
+
+    Gets the noise statistics from a raw data file. It won't parse the whole file, instead it will extract 2 chunks, one 
+    from t=duration to t=duration*2 and another from t=end of recording-duration*2 to t=end of recording-duration.
+    Then computes: the peak to peak, min, max, median and absolute median deviation of those chunks.
+
+    This is useful just to compare the start and end of the recording or to have ballpark estimations of these values. 
+    For more accurate measurements split the recording in chunks of e.g. 1 second, compute it for the entire file, then average and std.
+    This will max if there are artifacts in the chunks. 
+
+    Joao Couto - labdata 2024
+    '''
+    
+    filepath = Path(filepath)
+    if str(filepath).endswith('.cbin'):
+        from mtscomp import decompress
+        data = decompress(filepath) #,filepath.with_suffix('.ch'))
+    elif str(filepath).endswith('.bin'):
+        from spks.spikeglx_utils import load_spikeglx_binary
+        data,meta = load_spikeglx_binary(filepath)
+    else:
+        raise ValueError(f'Could not handle extension: {filepath}')
+    if data.shape[0]/sampling_rate < duration:
+        duration = np.floor((data.shape[0]/sampling_rate)/2)
+    # read the head and tail data
+    head_data = np.array(data[int(sampling_rate*duration):int(sampling_rate*duration)*2],dtype=np.float32)*gain
+    tail_data = np.array(data[-int(sampling_rate*duration)*2:-int(sampling_rate*duration)],dtype = np.float32)*gain
+    dd = [head_data,tail_data]
+    res = dict(channel_peak_to_peak = np.zeros((len(channel_indices),len(dd))),
+                 channel_median = np.zeros((len(channel_indices),len(dd))),
+                 channel_mad = np.zeros((len(channel_indices),len(dd))),
+                 channel_max = np.zeros((len(channel_indices),len(dd))),
+                 channel_min = np.zeros((len(channel_indices),len(dd))))
+    from scipy.stats import median_abs_deviation
+    for i,d in enumerate(dd):
+        res['channel_mad'][:,i] = median_abs_deviation(d[:,channel_indices],axis = 0) 
+        res['channel_max'][:,i] = np.max(d[:,channel_indices],axis = 0) 
+        res['channel_min'][:,i] = np.min(d[:,channel_indices],axis = 0) 
+        res['channel_median'][:,i] = np.median(d[:,channel_indices],axis = 0) 
+        res['channel_peak_to_peak'][:,i] = res['channel_max'][:,i]-res['channel_min'][:,i]
+    return res
+
+def extract_events_from_nidq(paths):
+    '''
+    Extracts the events from the nidq files (spikeglx) and formats it so they can be inserted in the database.
+    '''
+    from spks.spikeglx_utils import load_spikeglx_binary,read_spikeglx_meta
+    from spks.sync import unpackbits_gpu
+
+    nidqfilepath = list(filter(lambda x: '.bin' in x.suffix ,paths))
+    if not len(nidqfilepath):
+        # didnt find the files, try compressed
+        nidqfilepath = list(filter(lambda x: '.cbin' in x.suffix ,paths))
+        if not len(nidqfilepath):
+            raise(OSError(f'Could not find the nidaq file path for {paths}.'))
+    nidqfilepath = nidqfilepath[0]
+    # load the nidq file
+    if str(nidqfilepath).endswith('.bin'):
+        dat,meta = load_spikeglx_binary(nidqfilepath)
+    else:
+        from mtscomp import decompress
+        dat = decompress(nidqfilepath)
+        meta = read_spikeglx_meta(nidqfilepath.with_suffix('.meta'))
+    # read the sync channel
+    onsets,offsets = unpackbits_gpu(dat[:,-1])
+    
+    stream_name = 'nidq'
+    events = []
+    for k in onsets.keys():
+        o = onsets[k]
+        f = offsets[k]
+        ts = np.hstack([o,f])
+        v = np.hstack([np.array(o)*0+1,np.array(f)*0]).astype('uint8')
+        ii = np.argsort(ts)
+        events.append(dict(event_name = k,
+                           event_timestamps = ts[ii]/meta['sRateHz'], # in seconds
+                           event_values =  v[ii])) # store the onsets and offsets
+    return events, dat