labdata 0.0.23__tar.gz → 0.0.25__tar.gz

{labdata-0.0.23 → labdata-0.0.25}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: labdata
-Version: 0.0.23
+Version: 0.0.25
 Summary: Package to manage data in experimental neuroscience labs
 Author-email: Joao Couto <jpcouto@gmail.com>
 Project-URL: Homepage, https://github.com/jcouto/labdata
@@ -23,7 +23,7 @@ Requires-Dist: dash; extra == "dashboard"
 Requires-Dist: dash_bootstrap_components; extra == "dashboard"
 Requires-Dist: dash-extensions; extra == "dashboard"
 Requires-Dist: plotly; extra == "dashboard"
-Requires-Dist: streamlit>1.34; extra == "dashboard"
+Requires-Dist: streamlit>1.54; extra == "dashboard"
 Requires-Dist: streamlit-bokeh; extra == "dashboard"
 Requires-Dist: scipy; extra == "dashboard"
 Requires-Dist: scikit-image; extra == "dashboard"

{labdata-0.0.23 → labdata-0.0.25}/labdata/__init__.py

@@ -1,4 +1,4 @@
-VERSION = "0.0.23"
+VERSION = "0.0.25"
 
 from .utils import *
 from .copy import copy_to_upload_server

{labdata-0.0.23 → labdata-0.0.25}/labdata/cli.py

@@ -566,7 +566,7 @@ Available analysis are
                 if log is None:
                     log = line
                 else:
-                    if log.endswith('\n'):
+                    if log.endswith('\n') or log.endswith('\r'):
                         log += line
                     else:
                         log += '\n' + line

{labdata-0.0.23 → labdata-0.0.25}/labdata/compute/caiman.py

@@ -163,6 +163,8 @@ class CaimanCompute(Suite2pCompute):
         else:
             frame_rate = (self.schema.Miniscope() & self.dataset_key).fetch1('frame_rate')
             is_two_photon = False
+        if len((self.schema.File & dset).check_if_files_local()[0]) == 0:
+            self.files_existed = False
         dat = dset.open()
         parameters = (self.schema.CellSegmentationParams & f'parameter_set_num = {self.parameter_set_num}').fetch1()
 
@@ -271,6 +273,9 @@ class CaimanCompute(Suite2pCompute):
         selectiondict = []
         for icell,roi in enumerate(roi_masks):
             roi_pixels, roi_pixels_values = get_roi_pixels(roi)
+            if len(roi_pixels)>5000:
+                print(f"[CaimanCompute] {icell} had an roi with {len(roi_pixels)} pixels. Skipping.")
+                continue
             roidict.append(dict(planekey,
                                 roi_num = icell,
                                 roi_pixels = roi_pixels,
@@ -320,6 +325,9 @@ class CaimanCompute(Suite2pCompute):
             print(f'[{self.name} job] Removing the temporary folder.')
             import shutil
             shutil.rmtree(temporary_folder)
+            if not self.files_existed:
+                for f in (self.schema.File & dset).get():
+                    os.unlink(f)
         else:
             print(f'[{self.name} job] Kept the temporary folder {temporary_folder}.')
 

{labdata-0.0.23 → labdata-0.0.25}/labdata/compute/ephys.py

@@ -21,7 +21,7 @@ class SpksCompute(BaseCompute):
         self.file_filters = ['.ap.']
         # default parameters
         self.parameters = dict(algorithm_name = 'spks_kilosort4.0',
-                               motion_correction = True,
+                               motion_correction = 1,
                                low_pass = 300.,
                                high_pass = 13000.)
         self.parameter_keys = ['motion_correction','low_pass','high_pass','thresholds','remove_cross_duplicates',
@@ -151,7 +151,7 @@ class SpksCompute(BaseCompute):
             if not args.thresholds is None:
                 self.parameters['thresholds'] = args.thresholds
             if self.parameters['motion_correction'] < 2:
-                self.parameters['motion_correction'] = bool(self.parameters['motion_correction'])
+                self.parameters['motion_correction'] = self.parameters['motion_correction']
         self.probe = args.probe
         if not self.probe is None:
             self.parameters['probe'] = int(self.probe) # submit a single probe
@@ -188,8 +188,8 @@ class SpksCompute(BaseCompute):
             datasets += (self.schema.EphysRecording()& k).proj('subject_name','session_name','dataset_name').fetch(as_dict = True)
             
         if not parameter_set_num is None:
-            datasets = ((self.schema.EphysRecording() & datasets) -
-                        (self.schema.SpikeSorting() & f'parameter_set_num = {parameter_set_num}')).proj(
+            datasets = (self.schema.EphysRecording & ((self.schema.EphysRecording.ProbeSetting() & datasets) -
+                        (self.schema.SpikeSorting() & f'parameter_set_num = {parameter_set_num}'))).proj(
                             'subject_name',
                             'session_name',
                             'dataset_name').fetch(as_dict = True)
@@ -314,7 +314,9 @@ class SpksCompute(BaseCompute):
                 # send the files to AWS # this will skip the raw data binary files
                 dataset = dict(**self.dataset_key[0])
                 dataset['dataset_name'] = f'spike_sorting/imec{probe_num}/{self.parameter_set_num}/{self.parameters["algorithm_name"]}'
-                src = [a for a in Path(results_folder).rglob('*') if not str(a).endswith('.bin')] # get the files minus the .bin
+                src = [a for a in Path(results_folder).rglob('*') if not (str(a).endswith('.bin') or 
+                                                                          str(a).endswith('.dat') or
+                                                                          str(a).endswith('.hdf5'))] # get the files minus the .bin/.dat
                 new_src = [Path(prefs['local_paths'][0])/a for a in self.schema.AnalysisFile().generate_filepaths(src,dataset)]
                 from tqdm.auto import tqdm
                 import shutil
@@ -445,7 +447,7 @@ class SpksCompute(BaseCompute):
                           shuffle = True)
             from tqdm.auto import tqdm
             # print(f'Collecting waveforms and saving for dataset:{idset}.')
-            # save these to zarr to be compressed faster
+            # save these to zarr to be compressed faster?
             if self.use_hdf5: # zarr not implemented yet.
                 import h5py as h5
                 with h5.File(Path(results_folder)/f'waveforms{idset}.hdf5','w') as wavefile:
@@ -475,6 +477,7 @@ class SpksCompute(BaseCompute):
             # insert the syncs
             if len(event):
                 # Add stream
+                print(f'Inserting {len(event)} events.')
                 self.schema.DatasetEvents.insert1(dict(dataset_key,
                                                        stream_name = stream_name),
                                                   skip_duplicates = True,
@@ -504,10 +507,10 @@ class SpksCompute(BaseCompute):
                                             offset_samples = offset_samples - offsets[0],
                                             segment = np.array(dat[offset_samples : offset_samples + nsamples])))
             del dat
-            self.set_job_status(job_log = f'Completed {base_key}')
-            from labdata.schema import UnitMetrics
-            # limit number of jobs because of memory constraints
-            self.schema.UnitMetrics.populate(base_key, processes = int(max(1,np.ceil(n_jobs/2))))
+        self.set_job_status(job_log = f'Completed {base_key}')
+        from labdata.schema import UnitMetrics
+        # limit number of jobs because of memory constraints
+        self.schema.UnitMetrics.populate(base_key, display_progress = True, processes = int(max(1,np.ceil(n_jobs/2))))
     
     def extract_waveforms(self,udict, nchannels, results_folder, n_pre_samples,n_jobs, offsets):
         # extract the waveforms
@@ -575,7 +578,8 @@ def get_waveforms_from_binary(binary_file,
                               n_jobs = 8):
     from tqdm.auto import tqdm
     from spks.io import map_binary
-    dat = map_binary(binary_file,nchannels = binary_file_nchannels) 
+    # get the spikes for kilosort from the temp_wh.dat file if available.
+    dat = map_binary(binary_file, nchannels = binary_file_nchannels) 
     # return as generator to avoid having to use huge amounts of memory.
     res = Parallel(backend='loky',n_jobs=n_jobs,return_as = 'generator')(delayed(get_spike_waveforms)(
         dat,

{labdata-0.0.23 → labdata-0.0.25}/labdata/compute/suite2p.py

@@ -3,7 +3,7 @@ from .utils import BaseCompute
 
 class Suite2pCompute(BaseCompute):
     container = 'labdata-suite2p'
-    cuda = False
+    cuda = True
     name = 'suite2p'
     url = 'http://github.com/mouseland/suite2p'
     def __init__(self,job_id, project = None, allow_s3 = None, **kwargs):
@@ -163,8 +163,9 @@ class Suite2pCompute(BaseCompute):
         return datasets
         
     def _compute(self):
-        from numba import set_num_threads # control the number of threads used by suite2p 
-        set_num_threads(DEFAULT_N_JOBS)
+        # control the number of threads used by suite2p this colides with FISSA, for now don't limit.
+        #from numba import set_num_threads 
+        #set_num_threads(DEFAULT_N_JOBS)
         from threadpoolctl import threadpool_limits
         import suite2p
         version = f'suite2p{suite2p.version}'
@@ -173,8 +174,11 @@ class Suite2pCompute(BaseCompute):
             version+= f';fissa{fissa.__version__}'
         
         seskeys = (self.schema.TwoPhoton() & self.dataset_key).proj().fetch(as_dict = True)
+        self.files_existed = True
+        if len((self.schema.File & (self.schema.TwoPhoton & seskeys)).check_if_files_local()[0]) == 0:
+            self.files_existed = False
         if len(seskeys):
-            rec_files = (self.schema.File & (self.schema.TwoPhoton & seskeys)).get()
+            rec_filesorig = (self.schema.File & (self.schema.TwoPhoton & seskeys)).get()
             ses_par = (self.schema.TwoPhoton() & seskeys).fetch(as_dict = True)
         else:
             raise(NotImplemented("Only two-photon segmentation is implemented using this compute."))
@@ -188,20 +192,20 @@ class Suite2pCompute(BaseCompute):
         savedir = Path(prefs['scratch_path'])/f'suite2p_temporary_{rand}'
         savedir.mkdir(exist_ok=True,parents=True)
 
-        ops = suite2p.default_ops()
+        ops = suite2p.default_settings()
         for p in self.parameters.keys():
             if p in ops.keys():
                 ops[p] = self.parameters[p]
 
         # handle different file types
-        sbx_files = [f for f in rec_files if str(f).endswith('.sbx')]
-        zarr_files = [f for f in rec_files if str(f).endswith('.zarr.zip')]
+        sbx_files = [f for f in rec_filesorig if str(f).endswith('.sbx')]
+        zarr_files = [f for f in rec_filesorig if str(f).endswith('.zarr.zip')]
         if len(zarr_files):
             print('Converting zarr file to tiff.')
             rec_files = [savedir/'rawdata.tif']
             from tifffile import TiffWriter
             with TiffWriter(str(rec_files[0]), bigtiff=True, append=True) as tif:
-                from tqdm import tqdm 
+                from tqdm.auto import tqdm 
                 for ifile,zarrfile in enumerate(zarr_files):
                     fd = open_zarr(zarrfile)
                     for o,f in tqdm(chunk_indices(len(fd),512),
@@ -213,7 +217,7 @@ class Suite2pCompute(BaseCompute):
             ops['input_format'] = 'sbx'
             rec_files = sbx_files
         else:
-            raise(ValueError(f'No valid file format found in {rec_files}.'))
+            raise(ValueError(f'No valid file format found in {rec_filesorig}.'))
         
         ops['fs'] = fs
         ops['nplanes'] = nplanes
@@ -221,15 +225,19 @@ class Suite2pCompute(BaseCompute):
         ops['fast_disk'] = str(savedir)
         ops['save_folder'] = str(savedir/'suite2p')
         ops['combined'] = False # do not combine planes
-        db = {
-            'data_path': [s.parent for s in rec_files],
-        }
+        ops['save_path0'] = str(savedir/'suite2p')
+        ops['data_path'] = [s.parent for s in rec_files]
+        ops['file_list'] = rec_files
+        ops['reg_tif'] = True
         for o in ops.keys():
-            print(f'    {o} : {ops[o]}')
-        
+            print(f'[suite2p]    {o} : {ops[o]}', flush=True)
+
         with threadpool_limits(limits=DEFAULT_N_JOBS, user_api='blas'):
-            suite2p.run_s2p(ops=ops, db = db)
-        
+            suite2p.run_s2p(settings=dict(fs = ops['fs'],
+                                          tau = ops['tau'],
+                                          diameter = ops['diameter'],
+                                          torch_device = ops['torch_device']),
+                            db = ops)
         from labdata.stacks import compute_projections
         planefolders = natsorted([a.parent for a in savedir.rglob('*F.npy') 
                                   if not 'combined' in str(a)])
@@ -268,6 +276,22 @@ class Suite2pCompute(BaseCompute):
                                  roi_pixels_values = s['lam']))
             # extract and denoise with FISSA!
             if self.parameters['use_fissa']:
+                # this is a work around. Save the binary to a tiff file and extract with low-memory mode
+                if str(rec_files[0]).endswith('rawdata.tif'):
+                    if rec_files[0].exists():
+                        os.unlink(rec_files[0])
+                        print('Deleting the previously generated tiff file.')
+                #correctedfname = savedir/'motion_corrected'
+                #correctedfname.mkdir(exist_ok=True)
+                #from tifffile import imwrite
+                #from tqdm.auto import tqdm
+                #chunks = chunk_indices(len(binary),1056)
+                #chunkfiles = []
+                #for ichunk,(o,f) in tqdm(enumerate(chunks), total = len(chunks),
+                #                    desc = f"Concatenating motion corrected data:"):
+                #    chunkfiles.append(correctedfname/'chunk{ichunk}.tif')
+                #    imwrite(chunkfiles[-1],np.array(binary[o:f]).reshape((-1,*binary.shape[-2:])))
+                #with threadpool_limits(limits=DEFAULT_N_JOBS, user_api='blas'):
                 dff,spks = extract_dff_fissa(stats,binary,dims,
                                              batch_size=ops['batch_size'],
                                              tau=ops['tau'],
@@ -286,8 +310,14 @@ class Suite2pCompute(BaseCompute):
                              dims = [a for a in dims]))
                 cellcount[isession] += len(rois) # count cells in all planes
                 projdicts.extend([dict(planekey, proj_name = n,
-                                       proj_im = pi) for n,pi in zip(['mean','std','max','lcorr'],
-                                                                     [mean_proj,std_proj,max_proj,corr_proj])])
+                                       proj_im = pi) for n,pi in zip(['mean',
+                                                                      'std',
+                                                                      'max',
+                                                                      'lcorr'],
+                                                                     [mean_proj,
+                                                                      std_proj,
+                                                                      max_proj,
+                                                                      corr_proj])])
                 for icell,roi in enumerate(rois):
                     roidicts.append(dict(planekey,**roi))
                     if not dff is None:
@@ -320,23 +350,40 @@ class Suite2pCompute(BaseCompute):
         #     CellSegmentation.MotionCorrection.insert1(dict(planekey,
         #                                                    motion_block_size = 0,
         #                                                    displacement = motion))
-        self.schema.CellSegmentation.Projection.insert(projdicts,allow_direct_insert = True)
-        self.schema.CellSegmentation.ROI.insert(roidicts ,allow_direct_insert = True)
+        self.schema.CellSegmentation.Projection.insert(projdicts,
+                                                       allow_direct_insert = True)
+        self.schema.CellSegmentation.ROI.insert(roidicts,
+                                                allow_direct_insert = True)
         # Insert traces in parallel to prevent timeout errors from mysql
-        
-        parallel_insert(self.schema.schema_project,'CellSegmentation.Traces',tracesdicts, n_jobs = DEFAULT_N_JOBS,
-                        skip_duplicates = True, ignore_extra_fields = True)
-        parallel_insert(self.schema.schema_project,'CellSegmentation.RawTraces',rawtracesdicts, n_jobs = DEFAULT_N_JOBS,
-                        skip_duplicates = True, ignore_extra_fields = True)
-        parallel_insert(self.schema.schema_project,'CellSegmentation.Deconvolved',deconvdicts, n_jobs = DEFAULT_N_JOBS,
-                        skip_duplicates = True, ignore_extra_fields = True)                        
-        self.schema.CellSegmentation.Selection.insert(selectiondicts,allow_direct_insert = True)
+        parallel_insert(self.schema.schema_project,
+                        'CellSegmentation.Traces',
+                        tracesdicts,
+                        n_jobs = DEFAULT_N_JOBS,
+                        skip_duplicates = True,
+                        ignore_extra_fields = True)
+        parallel_insert(self.schema.schema_project,
+                        'CellSegmentation.RawTraces',
+                        rawtracesdicts,
+                        n_jobs = DEFAULT_N_JOBS,
+                        skip_duplicates = True,
+                        ignore_extra_fields = True)
+        parallel_insert(self.schema.schema_project,
+                        'CellSegmentation.Deconvolved',
+                        deconvdicts,
+                        n_jobs = DEFAULT_N_JOBS,
+                        skip_duplicates = True,
+                        ignore_extra_fields = True)                        
+        self.schema.CellSegmentation.Selection.insert(selectiondicts,
+                                                      allow_direct_insert = True)
         if not self.keep_intermediate:
-            print(f'[{self.name} job] Removing the temporary folder.')
+            print(f'[{self.name} job] Removing the temporary folder {savedir}.')
             import shutil
             shutil.rmtree(savedir)
+            if not self.files_existed:
+                for f in rec_filesorig:
+                    os.unlink(f)
         else:
-            print(f'[{self.name} job] Kept the temporary folder {temporary_folder}.')
+            print(f'[{self.name} job] Kept the temporary folder {savedir}.')
 
 
 def extract_dff_fissa(stats,binary,dims,fs, batch_size,tau):
@@ -361,13 +408,14 @@ def extract_dff_fissa(stats,binary,dims,fs, batch_size,tau):
     import fissa
     from tqdm.auto import tqdm
     roiidx = np.where([r.sum()>4 for r in tqdm(rois,desc = 'Using only cells with more than 4 pixels') ])[0]
-    fissa_res = fissa.Experiment(images=[binary],rois=[[rois[i] for i in roiidx]],
+    fissa_res = fissa.Experiment(images = [binary],
+                                 rois = [[rois[i] for i in roiidx]],
                                  ncores_separation = DEFAULT_N_JOBS,
-                                 ncores_preparation = DEFAULT_N_JOBS)
+                                 ncores_preparation = DEFAULT_N_JOBS,
+                                 verbosity = 2)
 
     res = fissa_res.separate()
     fissa_res.calc_deltaf(freq=fs)
-
     dff = np.zeros((num_rois,
                     len(fissa_res.deltaf_result[0][0][0])),dtype = 'float32')
     for iroi,r in zip(roiidx,fissa_res.deltaf_result):

{labdata-0.0.23 → labdata-0.0.25}/labdata/compute/utils.py

@@ -146,7 +146,7 @@ def run_analysis(target, jobids, compute_obj, project = None):
 
                 if container_file.exists():
 
-                    cmd += ' 2>&1 | ' +  run_on_apptainer(container_file,
+                    cmd += ' | ' +  run_on_apptainer(container_file,
                                                      command = f'labdata2 logpipe {jid}',
                                                      dry_run = True)
                 else:
@@ -223,7 +223,7 @@ def run_analysis(target, jobids, compute_obj, project = None):
                                            cuda = compute_obj.cuda,
                                            dry_run = True)
                     # generate slurm cmd and launch
-                    cmd += ' 2>&1 | ' + run_on_apptainer(container_file,
+                    cmd += ' | ' + run_on_apptainer(container_file,
                                                   command = f'labdata2 logpipe {j}',
                                                   dry_run = True)
                     opts = dict()

labdata-0.0.25/labdata/dashboard/compute.py

@@ -0,0 +1,112 @@
+import pandas as pd
+import numpy as np
+import streamlit as st
+
+STATES = ['COMPLETED', 'WORKING', 'WAITING', 'FAILED', 'CANCELED','ON SERVER']
+STATE_COLORS = ['#01D186', '#48CAE4', '#F3A21C', '#FF5733', '#F3A21C','#F3A21C']
+
+def _jobs_panel(table, status_field, target_field, cols, delete_key):
+
+    import datajoint as dj
+    import altair as alt
+
+    rows = table.fetch(as_dict=True)
+    if not rows:
+        st.write('### There are no jobs in the table')
+        return
+
+    df = pd.DataFrame(rows)[[c for c in cols if c in pd.DataFrame(rows).columns]]
+
+    # counts via aggr
+    counts = {r[status_field]: r['n'] for r in
+              dj.U(status_field).aggr(table, n='count(*)').fetch(as_dict=True)}
+    metric_cols = st.columns(len(STATES))
+    for col, t in zip(metric_cols, STATES):
+        col.metric(t, counts.get(t, 0))
+
+    # bar chart per target queue
+    df = df.set_index('job_id')
+    if target_field in df.columns:
+        targets = [d if d is not None else 'unknown' for d in df[target_field].values]
+        df[target_field] = targets
+        qcounts = []
+        for t in np.unique(targets):
+            status_vals = df[status_field].values
+            target_vals = df[target_field].values
+            for tt, c in zip(STATES, STATE_COLORS):
+                qcounts.append({'Task status': f'{tt}|{t}',
+                                'Jobs': int(sum(1 for a, tv in zip(status_vals, target_vals)
+                                               if tt in a and tv == t)),
+                                'color': c})
+        qcounts = [q for q in qcounts if q['Jobs'] > 0]
+        ch = (alt.Chart(pd.DataFrame(qcounts)).mark_bar().encode(
+            y='Task status:N',
+            x='Jobs:Q',
+            color=alt.Color('color:N').scale(None)
+        ).configure_axis(labelLimit=3000))
+        st.altair_chart(ch, width="stretch")
+
+    # filtered table
+    selected_states = st.multiselect('Select status:', STATES, default=STATES[1:], key=f'{delete_key}_filter')
+    idx = np.zeros(len(df), dtype=bool)
+    for s in selected_states:
+        idx = idx | np.array([s in a for a in df[status_field].values])
+
+    display_df = df.iloc[idx].copy()
+    display_df.insert(0, 'Select', False)
+    edited = st.data_editor(
+        display_df,
+        column_config={'Select': st.column_config.CheckboxColumn(required=True)},
+        disabled=[c for c in display_df.columns if c != 'Select'],
+        width="stretch",
+        key=f'{delete_key}_editor',
+    )
+    selected_jobs = edited[edited['Select'] == True].index.tolist()
+    if selected_jobs:
+        st.caption(f'{len(selected_jobs)} job(s) selected')
+        if st.button('Delete selected', type='primary', key=f'{delete_key}_btn'):
+            st.session_state[f'{delete_key}_pending'] = selected_jobs
+        if st.session_state.get(f'{delete_key}_pending'):
+            jobs_to_delete = st.session_state[f'{delete_key}_pending']
+            st.warning(f'Are you sure you want to delete {len(jobs_to_delete)} job(s)? This cannot be undone and scheduled jobs might fail.')
+            col_yes, col_no, _ = st.columns([1, 1, 4])
+            if col_yes.button('Yes, delete', type='primary', key=f'{delete_key}_yes'):
+                (table & [dict(job_id=j) for j in jobs_to_delete]).delete(safemode=False)
+                st.session_state.pop(f'{delete_key}_pending', None)
+                st.success(f'Deleted {len(jobs_to_delete)} job(s).')
+                st.rerun()
+            if col_no.button('Cancel', key=f'{delete_key}_no'):
+                st.session_state.pop(f'{delete_key}_pending', None)
+                st.rerun()
+
+
+def compute_tab(schema=None):
+    if schema is None:
+        from labdata import load_project_schema
+        schema = load_project_schema()
+
+    tab_compute, tab_upload = st.tabs(['Compute tasks', 'Upload jobs'])
+
+    with tab_compute:
+        _jobs_panel(
+            table=schema.ComputeTask,
+            status_field='task_status',
+            target_field='task_target',
+            cols=['job_id', 'task_name', 'task_status', 'task_host', 'task_target',
+                  'task_waiting', 'task_log', 'subject_name', 'session_name',
+                  'task_starttime', 'task_endtime'],
+            delete_key='compute',
+        )
+
+    with tab_upload:
+        all_projects = st.toggle('Show all projects', value=False, key='upload_all_projects')
+        upload_table = schema.UploadJob if all_projects else schema.UploadJob & dict(project_name=schema.schema_project)
+        _jobs_panel(
+            table=upload_table,
+            status_field='job_status',
+            target_field='job_host',
+            cols=['job_id', 'job_status', 'job_host', 'job_rule', 'job_waiting',
+                  'job_log', 'subject_name', 'session_name', 'dataset_name',
+                  'upload_storage', 'job_starttime', 'job_endtime'],
+            delete_key='upload',
+        )