labdata 0.0.3__py3-none-any.whl

labdata/schema/twophoton.py

@@ -0,0 +1,142 @@
+from .general import *
+from .procedures import *
+from .histology import *
+
+@dataschema
+class TwoPhoton(dj.Imported):
+    definition = '''
+    -> Dataset
+    ---
+    n_planes               : smallint            # number of planes
+    n_channels             : smallint            # number of channels
+    n_frames               : int                 # duration of the recording
+    width                  : int                 # width of each frame
+    height                 : int                 # height of each frame
+    frame_rate             : double              # frame rate
+    -> File                                      # path to the stack
+    magnification = NULL   : double              # magnification
+    objective_angle = NULL : double              # angle
+    objective = NULL       : varchar(32)         # objective
+    um_per_pixel = NULL    : blob                # XY scale conversion factors
+    scanning_mode = NULL   : varchar(32)         # bidirectional or unidirectional
+    pmt_gain = NULL        : blob                # pmt gains
+    imaging_software       : varchar(32)         # software and version
+    -> [nullable] Atlas.Region                   # target region (or the center of the imaging plane)
+    '''
+
+    class Plane(dj.Part):
+        definition = '''
+        -> master
+        plane_num               : smallint       # probe number
+        ---
+        plane_depth = NULL      : float          # depth from surface
+        '''
+    
+    def open(self):
+        # download and open a stack
+        if len(self) != 1:
+            raise(ValueError(f'Select only one dataset {self.proj().fetch(as_dict = True)}.'))
+        fname = (File() & self).get()[0]
+        return open_zarr(fname)
+
+@analysisschema
+class CellSegmentationParams(dj.Manual):
+    definition = '''
+    parameter_set_num      : int            # number of the parameters set
+    ---
+    algorithm_name               : varchar(64)    # cell segmentation algorithm
+    parameter_description = NULL : varchar(256)   # description or specific use case
+    parameters_dict              : varchar(2000)  # parameters json formatted dictionary "json.dumps"
+    code_link = NULL             : varchar(300)   # link to the code
+    '''
+
+@analysisschema
+class CellSegmentation(dj.Imported):
+    definition = '''
+    -> Dataset
+    -> CellSegmentationParams
+    ---
+    algorithm_version             : varchar(28)     # version of the algorithm
+    num_rois                      : int             # number of segmented ROIs
+    crop_region = NULL            : blob            # Region to crop
+    segmentation_datetime = NULL  : datetime        # date that the algorith was ran
+    -> [nullable] AnalysisFile                      # results file (optional)
+     '''
+
+    class Plane(dj.Part):
+        definition = '''
+        -> master
+        plane_num                 : smallint
+        ---
+        plane_num_rois           : int
+        '''
+    
+    class MotionCorrection(dj.Part):
+        definition = '''
+        -> master.Plane
+        ---
+        motion_block_size = 0     : int             # 0 is rigid
+        displacement              : longblob        # storage of the motion displacement 
+        '''
+    class Projection(dj.Part):
+        definition = '''
+        -> master.Plane
+        ---
+        proj_name                 : varchar(16)
+        proj_im                   : longblob
+        '''
+    class ROI(dj.Part):
+        definition = '''
+        -> master.Plane
+        roi_num                   : int
+        ---
+        roi_pixels                : blob
+        roi_pixels_values = NULL  : blob
+        neuropil_pixels = NULL    : blob
+        '''
+    class RawTraces(dj.Part):
+        definition = '''
+        -> master.ROI
+        ---
+        f_trace                   : longblob
+        f_neuropil = NULL         : longblob 
+        '''    
+    class Traces(dj.Part):
+        definition = '''
+        -> master.ROI
+        ---
+        dff                       : longblob 
+        '''
+    class Deconvolved(dj.Part):
+        definition = '''
+        -> master.ROI
+        ---
+        deconv                    : longblob
+        '''
+    
+    class Selection(dj.Part):
+        definition = '''
+        -> master.ROI
+        selection_method = 'auto' : varchar(12)
+        ---
+        selection                 : smallint      # 0 not a cell, 1 is a cell
+        likelihood = NULL         : float
+        '''
+    
+@analysisschema
+class SegmentationROIMetrics(dj.Computed):
+    definition = '''
+    -> CellSegmentation.ROI
+    ---
+    roi_center               : blob
+    radius = NULL            : float
+    snr = NULL               : float
+    aspect_ratio = NULL      : float
+    skewness = NULL          : float
+    std = NULL               : float
+    noise_level = NULL       : float 
+    '''
+
+    def make(self, key):
+        # 90th percentile/np.median(np.abs(np.diff(dff))) / np.sqrt(frame_rate)?
+        return

labdata/schema/utils.py

@@ -0,0 +1,25 @@
+from ..utils import *
+# some functions used in the schema imports
+
+__all__ = ['read_camlog']
+
+def read_camlog(log):
+    '''
+    Adapted from github.com/jcouto/labcams 
+    '''
+    
+    logheaderkey = '# Log header:'
+    comments = []
+    with open(log,'r',encoding = 'utf-8') as fd:
+        for line in fd:
+            if line.startswith('#'):
+                line = line.strip('\n').strip('\r')
+                comments.append(line)
+                if line.startswith(logheaderkey):
+                    columns = line.strip(logheaderkey).strip(' ').split(',')
+    camlog = pd.read_csv(log, delimiter = ',',
+                         header = None,
+                         comment = '#',
+                         engine = 'c')
+    return comments,camlog
+

labdata/schema/video.py

@@ -0,0 +1,243 @@
+from .general import *
+
+@dataschema
+class DatasetVideo(dj.Manual):
+    definition = '''
+    -> Dataset
+    video_name           : varchar(56)
+    ---
+    frame_times = NULL   : longblob
+    frame_rate = NULL    : float
+    n_frames = NULL      : float
+    '''
+
+    class File(dj.Part):
+        definition = '''
+        -> master
+        -> File
+        '''
+
+    class Frame(dj.Part):
+        definition = '''
+        -> master
+        frame_num      : int
+        ---
+        frame          : longblob
+        '''
+
+######################################################
+############      POSE ESTIMATION      ###############
+######################################################
+@analysisschema
+class PoseEstimationLabelSet(dj.Manual):
+    definition = '''
+    pose_label_set_num    : int
+    ---
+    description = NULL    :  varchar(512)
+    -> [nullable] LabMember.proj(labeler = "user_name")
+    '''
+    class Frame(dj.Part):
+        definition = '''
+        -> master
+        -> DatasetVideo
+        frame_num : int
+        ---
+        frame     : longblob
+        '''
+    class Label(dj.Part):
+        definition = '''
+        -> master
+        -> DatasetVideo
+        frame_num  : int
+        label_name : varchar(54)
+        ---
+        x          : float
+        y          : float
+        z = NULL   : float 
+        '''
+    def export_labeling(self, model_num = None, bodyparts = None, disperse_labels = False, export_only_labeled = False):
+        '''
+        Exports labeling for PoseEstimation (for use with napari-deeplabcut)
+        '''
+        assert len(self) == 1, ValueError('PoseEstimationLabelSet, select only one set to export.')
+        k = self.proj().fetch1()
+        
+        if export_only_labeled:
+            frames = pd.DataFrame((PoseEstimationLabelSet()*PoseEstimationLabelSet.Frame() & (PoseEstimationLabelSet.Label() & k)).fetch())
+        else:
+            frames = pd.DataFrame((PoseEstimationLabelSet()*PoseEstimationLabelSet.Frame() & k).fetch())
+        frame_labels = pd.DataFrame((PoseEstimationLabelSet()*PoseEstimationLabelSet.Label() & k).fetch())
+        
+        if model_num is None:
+            folder = (Path(prefs['local_paths'][0])/'pose_estimation_models')/f'pose_label_set_num_{k["pose_label_set_num"]}'
+        else:
+            folder = (Path(prefs['local_paths'][0])/'pose_estimation_models')/f'model_{model_num}'
+        data_path = (folder / "labeled-data") / f'label_set_{k["pose_label_set_num"]}'
+        data_path.mkdir(parents=True, exist_ok=True)
+        if bodyparts is None:
+            bodyparts = np.unique(frame_labels.label_name.values)
+        from natsort import natsorted
+        bodyparts = natsorted(bodyparts) # this is an attempt to sort the labels
+        labeler = frames['labeler'].iloc[0]
+        from skimage.io import imsave
+        from tqdm import tqdm
+        todlc = []
+        for i,f in tqdm(enumerate(frames.frame_num.values),desc = "Exporting labeling dataset:",total = len(frames)):
+            im_name = 'im_{0:06d}_session{2}_frame{1:06d}'.format(i,f,frames.session_name.iloc[i])
+            for bpart in bodyparts:
+                t = (PoseEstimationLabelSet.Label & dict(
+                    pose_label_set_num = k['pose_label_set_num'],
+                    frame_num = f,
+                    label_name = bpart)).fetch()
+                x = np.nan
+                y = np.nan
+                if disperse_labels:
+                    if i == 0:
+                        x = i*20
+                        y = 100
+                if len(t):
+                    x = t['x'][0]
+                    y = t['y'][0]
+                todlc.append(dict(scorer = labeler,
+                                bodyparts = bpart,
+                                level_0 = 'labeled-data',
+                                level_1 = f'label_set_{k["pose_label_set_num"]}',
+                                level_2 = f'{im_name}.png',
+                                x = x,
+                                y = y))
+            fname = data_path/f'{im_name}.png'
+            if not fname.exists():
+                imsave(fname,frames.iloc[i].frame)
+        df = pd.DataFrame(todlc)
+        df = df.set_index(["scorer", "bodyparts","level_0","level_1","level_2"]).stack()
+        df.index.set_names("coords", level=-1, inplace=True)
+        df = df.unstack(["scorer", "bodyparts", "coords"])
+        df.index.name = None
+        df.to_hdf(data_path/f'CollectedData_{labeler}.h5',key='keypoints')
+        return data_path,frames,frame_labels
+
+    def update_labeling(self, labeling_file):
+        '''
+        (PoseEstimationLabelSet() & 'pose_label_set_num =3').update_labeling('filename.h5')
+
+        Updates the labels in the PoseEstimationLabelSet from a file.
+        Currently only DLC format is supported.
+
+        Reach out if you need other formats.
+         Joao Couto 2023
+        '''
+        dlcres = pd.read_hdf(labeling_file)
+        scorer = np.unique(dlcres.columns.get_level_values(0))[0]
+        bodyparts = np.unique(dlcres.columns.get_level_values(1))
+        frame_nums = [int(f.split('frame')[-1].strip('.png')) 
+                    for f in dlcres.reset_index()['level_2'].values]
+        frame_names = dlcres.reset_index()['level_2'].values
+        labels = []
+        from tqdm import tqdm
+        labels_to_insert = [] # insert the labels in parallel will be faster.
+        labels_to_delete = [] # need to delete all labels for a frame before adding the new ones
+        for iframe,frame_name in tqdm(enumerate(frame_names),desc = 'Updating labels',total = len(frame_names)):
+            frame_num = int(frame_name.split('frame')[-1].strip('.png'))
+            frame_key = dict(frame_num = frame_num)
+            if 'session' in frame_name: # get the session name so there are no conflicting frame numbers
+                 frame_key['session_name'] = frame_name.split('session')[-1].split('_frame')[0]
+            frame_key = (PoseEstimationLabelSet.Frame() & self.proj().fetch1() & frame_key).proj().fetch1()
+            labels_to_delete.extend((PoseEstimationLabelSet.Label() & frame_key).proj().fetch(as_dict = True))
+            for dlcname in bodyparts:
+                if np.isnan(dlcres[scorer][dlcname].iloc[iframe]['x']):
+                    continue # if it is NaN, don't add
+                if dlcres[scorer][dlcname].iloc[iframe]['x'] == 0 and dlcres[scorer][dlcname].iloc[iframe]['y'] == 0:
+                    continue # if the label is at 0,0  don't add
+                label = dict(dict(frame_key,label_name = dlcname),
+                            label_name = dlcname,
+                            x = dlcres[scorer][dlcname].iloc[iframe]['x'],
+                            y = dlcres[scorer][dlcname].iloc[iframe]['y'])
+                labels_to_insert.append(label)
+        (PoseEstimationLabelSet.Label() & labels_to_delete).delete(force = True) # ask the user to confirm
+        PoseEstimationLabelSet.Label.insert(labels_to_insert)
+                    
+
+@analysisschema
+class PoseEstimationModel(dj.Manual):
+    definition = '''
+    model_num      : int
+    ---
+    algorithm_name           : varchar(24)    # Algorithm for pose estimation
+    -> AnalysisFile                           # zipped model; no videos.
+    -> [nullable] PoseEstimationLabelSet 
+    parameters_dict = NULL   : varchar(2000)  # parameters json formatted dictionary
+    training_datetime = NULL : datetime
+    container_name = NULL    : varchar(64)    # Name of the container to use
+    code_link = NULL         : varchar(300)   # link to the github of the algorithm
+    '''
+    
+    def insert_model(self, model_num, 
+                     model_folder=None,
+                     pose_label_set_num = None,
+                     algorithm_name = None,
+                     parameters = None,
+                     training_datetime=None,
+                     container_name = None,
+                     code_link = None):    
+        import shutil
+        if training_datetime is None:
+            today = datetime.now()
+        else:
+            today = training_datetime
+        dataset_name = datetime.strftime(today,'%Y%m%d_%H%M%S')
+
+        # check if this model_number exists for another pose_label_set_num
+        allmodels = pd.DataFrame(PoseEstimationModel.fetch())
+        sel = allmodels[(allmodels.model_num.values == model_num) & (allmodels.pose_label_set_num.values != pose_label_set_num)]
+        if len(sel):
+            model_num = np.max(allmodels.model_num.values)+1
+                    
+        if model_folder is None:
+            model_folder = ((Path(prefs['local_paths'][0])/'pose_estimation_models')/f'{dataset_name}')/f'model_{model_num}'
+        filepath = ((Path(prefs['local_paths'][0])/'pose_estimation_models')/f'{dataset_name}')/f'model_{model_num}'
+        print(f'Creating archive {filepath}')
+        shutil.make_archive(filepath, 'zip', model_folder)
+        filepath = filepath.with_suffix('.zip')
+
+        key = AnalysisFile().upload_files([filepath],dataset = dict(subject_name = 'pose_estimation_models',
+                                                                  session_name = f'model_{model_num}',
+                                                                  dataset_name = dataset_name))
+        key = dict((AnalysisFile & key).proj().fetch1(),
+                          model_num = model_num,
+                          algorithm_name = algorithm_name,
+                          training_datetime = today,
+                          pose_label_set_num = pose_label_set_num,
+                          parameters_dict = json.dumps(parameters) if not parameters is None else None,
+                          container_name = container_name,
+                          code_link = code_link)
+        if not len(PoseEstimationModel & f'model_num = {model_num}'):
+            self.insert1(key)
+        else:
+            print(f'Model {model_num} already exists. Updating but keeping last version in AWS.')        
+            oldentry = (PoseEstimationModel & f'model_num = {model_num}').fetch1()
+            for k in key.keys():
+                if key[k] is None:
+                    key[k] = oldentry[k]
+            self.update1(key)
+        # need to add a model evaluation part table here
+    def get_model(self):
+        filepath = (AnalysisFile & self).get()
+        
+        filepath = filepath[0]
+        if not (filepath.parent/'config.yaml').exists():
+            import shutil 
+            shutil.unpack_archive(filepath,extract_dir = filepath.parent)
+        return filepath.parent/'config.yaml' # return the path to the config file.
+        
+@analysisschema
+class PoseEstimation(dj.Manual):
+    definition = '''
+    -> PoseEstimationModel
+    -> DatasetVideo
+    label_name : varchar(54)
+    ---
+    x          : longblob
+    y          : longblob
+    z = NULL   : longblob
+    likelihood : longblob
+    '''

labdata/stacks.py

@@ -0,0 +1,182 @@
+# files to process or visualize imaging stacks
+
+from .utils import *
+
+def downsample_stack(stack, ratio = [0.3,1,0.40625,0.40625],n_jobs = DEFAULT_N_JOBS,order = 1):
+    '''
+    Downsamples a 4D stack using interpolation.
+
+    smaller_stack = downsample_stack(stack, ratio = [0.3,1,0.40625,0.40625],n_jobs = DEFAULT_N_JOBS)
+    
+    '''
+    assert len(stack.shape) == 4, ValueError('Only downsampling 4D stacks is supported.')
+    from scipy.ndimage import zoom
+    from joblib import Parallel, delayed
+    from tqdm import tqdm
+    import numpy as np
+    assert len(stack.shape) == len(ratio), ValueError(f'Stack ({stack.shape}) and ratio {len(ratio)} need to match')
+    a = Parallel(n_jobs = n_jobs)(delayed(zoom)(
+        frame,np.array(ratio)[1:],order = order) 
+        for frame in tqdm(stack, 
+        total = stack.shape[0],desc = 'Downsampling inner dimensions:'))
+    na = np.stack(a)
+    if na.shape[1] == 1:
+        return na
+    del a
+    na = Parallel(n_jobs = n_jobs)(delayed(zoom)(
+        na[:,i],[ratio[0],1,1],order = order) 
+        for i in tqdm(range(na.shape[1]), 
+        total = stack.shape[1],desc = 'Downsampling outer dimensions:'))
+    return np.stack(na).transpose([1,0,2,3])
+
+def rotate_stack(stack, 
+                 anglez = 0, 
+                 angley = 0, 
+                 anglex = 0, 
+                 flip_x = False, 
+                 flip_y = False, 
+                 n_jobs = DEFAULT_N_JOBS,
+                 order = 1):
+    '''
+    Rotate a stack a 4d stack.
+
+    This can be used for instance to rotate a whole brain image. Channels are rotated in paralell, careful with the 
+    To get the angles: 
+    
+    Joao Couto - adapted from deeptrace (2023)
+    '''
+    from tqdm import tqdm
+    na = stack.transpose(1,0,2,3)
+    from scipy.ndimage import rotate
+    if anglex != 0.0:
+        na = Parallel(n_jobs = n_jobs)(delayed(rotate)(
+            s, angle = anglex,order = order,axes = [1,2],reshape = False) 
+            for s in tqdm(na, total = na.shape[0],desc = 'Rotating the x axis:'))
+        na = np.stack(na)
+    if angley != 0.0:
+        na = Parallel(n_jobs = n_jobs)(delayed(rotate)(
+            s, angle = angley,order = order,axes = [0,2],reshape = False) 
+            for s in tqdm(na, 
+            total = na.shape[0],desc = 'Rotating the y axis:'))
+        na = np.stack(na)
+    if anglez != 0.0:
+        na = Parallel(n_jobs = n_jobs)(delayed(rotate)(
+            s, angle = anglez,order = order,axes = [0,1],reshape = False) 
+            for s in tqdm(na, 
+            total = na.shape[0],desc = 'Rotating the z axis:'))
+        na = np.stack(na)
+    if flip_x:
+        na = na[:,:,:,::-1]
+    if flip_y:
+        na = na[:,:,::-1,:]
+    return na.transpose(1,0,2,3)
+
+def napari_open(stack,**kwargs):
+    '''
+    Example:
+
+    napari_open(stack.transpose(1,0,2,3),contrast_limits=[0,65000],channel_axis = 1,multiscale=False)
+
+    '''
+    import napari
+    napari.view_image(stack,**kwargs)
+
+class VideoStack():
+    '''
+    Class to read video file sequences.
+    '''
+    def __init__(self,files, use_fast_seek = False):
+        self.N = 0
+        self.W = None
+        self.H = None
+        self.files = files
+        self.file_offsets = []
+        self.current_file = None
+        self.current_offset = -1
+        self.use_fast_seek = use_fast_seek
+        self.cap = None
+        self._read_properties()
+        self.dtype = np.uint8
+
+
+
+    def _read_properties(self):
+        self.file_offsets = [0]
+        self.N = 0
+        self.W = None
+        self.H = None
+        import cv2
+        for f in self.files:
+            cap = cv2.VideoCapture(f)
+            N = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
+            self.N += N
+            self.file_offsets.append(N)
+            if self.W is None:
+                self.W  = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
+                self.H = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
+            if self.W != int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)):
+                raise(ValueError(f'Video {f} has a different width.'))
+            if self.H != int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)):
+                raise(ValueError(f'Video {f} has a different height.'))
+            cap.release()
+        self.shape = (self.N,self.H,self.W)
+        self.file_offsets = np.cumsum(self.file_offsets)
+    def __len__(self):
+        return self.shape[0]
+    
+    def __getitem__(self, *args):
+        index  = args[0]
+        idx1 = None
+        idx2 = None
+        if type(index) is tuple: # then look for 2 channels
+            if type(index[1]) is slice:
+                idx2 = range(index[1].start, index[1].stop, index[1].step)
+            else:
+                idx2 = index[1]
+            index = index[0]
+        if type(index) is slice:
+            idx1 = range(*index.indices(self.N))#start, index.stop, index.step)
+        elif type(index) in [int,np.int32, np.int64]: # just a frame
+            idx1 = [index]
+        else: # np.array?
+            idx1 = index
+        img = np.empty((len(idx1),*self.shape[1:]), dtype = self.dtype)
+        # print(img.shape,idx1)
+        for i,ind in enumerate(idx1):
+            img[i] = self._get_frame(ind)
+        if not idx2 is None:
+            return img[:,idx2].squeeze()
+        else:
+            return img[:].squeeze()
+
+    def _open_file(self,ifile):
+        if not self.cap is None:
+            self.cap.release()
+            self.cap = None
+        import cv2
+        self.cap = cv2.VideoCapture(self.files[ifile])
+        self.current_offset = 0
+        self.current_file = ifile
+        # print(f'Opened file {self.files[ifile]}.')
+        
+    def _get_frame(self,ind):
+        import cv2
+        ifile = np.where(self.file_offsets <= ind)[0][-1]
+        if not self.current_file == ifile:
+            self._open_file(ifile)
+        ind = ind - self.file_offsets[self.current_file]
+        if ind == 0:
+            self._open_file(ifile) # open twice if zero because we may need to reset for sequential access
+        if self.use_fast_seek and not self.current_offset == ind:
+            self.cap.set(cv2.CAP_PROP_POS_FRAMES,ind)
+            ret, frame = self.cap.read()
+            self._open_file(ifile)
+            return frame[...,0]
+        
+        while True:
+            ret, frame = self.cap.read()
+            self.current_offset += 1
+            if self.current_offset-1 == ind:
+                return frame[...,0]
+            elif (self.current_offset-1) > ind: # re-open the file
+                self._open_file(ifile)