labdata 0.1.1__tar.gz → 0.1.2__tar.gz

{labdata-0.1.1 → labdata-0.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: labdata
-Version: 0.1.1
+Version: 0.1.2
 Summary: Package to manage data in experimental neuroscience labs
 Author-email: Joao Couto <jpcouto@gmail.com>
 Project-URL: Homepage, https://github.com/jcouto/labdata

{labdata-0.1.1 → labdata-0.1.2}/labdata/__init__.py

@@ -1,4 +1,4 @@
-VERSION = "0.1.1"
+VERSION = "0.1.2"
 
 from .utils import *
 from .copy import copy_to_upload_server

{labdata-0.1.1 → labdata-0.1.2}/labdata/compute/caiman.py

@@ -22,7 +22,7 @@ class CaimanCompute(Suite2pCompute):
         # default parameters
         self.parameters = dict(algorithm_name = 'caiman')
         # will only store these in CellSegmentationParams
-        self.parameter_keys = ['pw_rigid','p','gSig','gSiz','merge_thr','rf','stride','tsub',
+        self.parameter_keys = ['pw_rigid','p','gSig','gSig_filt','max_shift','gSiz','merge_thr','rf','stride','tsub',
                                'ssub','nb','min_corr','min_pnr','ssub_B','ring_size_factor',
                                'min_SNR', 'rval_thr', 'use_cnn', 'detrendWin','quantileMin','denoise_dff']
         
@@ -50,12 +50,18 @@ class CaimanCompute(Suite2pCompute):
             parser.add_argument('-m','--pwrigid',
                                 action='store_true', default=False,
                                 help = "Piecewise-rigid registration")
+            parser.add_argument('--max-allowed-shift',
+                                action='store', default=[20,20], type = int, nargs = 2,
+                                help = "maximum allowed shift for motion correction")
             parser.add_argument('-p',
                                 action='store', default=1, type = int,
                                 help = "Order of the autoregressive system")
             parser.add_argument('-g','--gsig',
                                 action='store', default=[6,6], type = int, nargs = 2,
                                 help = "Expected halfwidth of the neurons in pixels")
+            parser.add_argument('--gsig-filt',
+                                action='store', default=None, type = int, nargs = 2,
+                                help = "Filter for 1photon data")
             parser.add_argument('--nb',
                                 action='store', default=0, type = int,
                                 help = "number of background components (rank) if positive, set to 0 for CNMFE")
@@ -113,8 +119,10 @@ class CaimanCompute(Suite2pCompute):
             args = parser.parse_args(arguments[1:])
             
             params = dict(pw_rigid = args.pwrigid,
+                          max_shifts = args.max_allowed_shifts,
                           p = int(args.p),
                           gSig = [int(a) for a in args.gsig], 
+                          gSig_filt = [int(a) for a in args.gsig_filt], 
                           gSiz = [int(a) for a in 2*np.array(args.gsig) + 1],
                           merge_thr = float(args.merge_thr),      
                           rf = int(args.rf),                    
@@ -143,18 +151,18 @@ class CaimanCompute(Suite2pCompute):
         temporary_folder = Path(prefs['scratch_path'])/f'caiman_temporary_{rand}'
         dset = (self.schema.Miniscope() & self.dataset_key)
         cnmfparams = {
-                'motion_correct' : True,
-                'method_init': 'corr_pnr',  # use this for 1 photon
-                'K': None, # for 1p                                
-                'nb': 0,             # number of background components (rank) if positive, set to 0 for CNMFE
-                'nb_patch': 0,
-                'low_rank_background': None,           # for 1p
-                'update_background_components': True,  # sometimes setting to False improve the results
-                'del_duplicates': True,                # whether to remove duplicates from initialization
-                'normalize_init': False,               # just leave as is
-                'center_psf': True,                    # True for 1p
-                'only_init': True,    # set it to True to run CNMF-E
-                'method_deconvolution': 'oasis'}       # could use 'cvxpy' alternatively
+            'motion_correct' : True,
+            'method_init': 'corr_pnr',             # use this for 1 photon
+            'K': None,                             # for 1p                                
+            'nb': 0,                               # number of background components (rank) if positive, set to 0 for CNMFE
+            'nb_patch': 0,
+            'low_rank_background': None,           # for 1p
+            'update_background_components': True,  # sometimes setting to False improve the results
+            'del_duplicates': True,                # whether to remove duplicates from initialization
+            'normalize_init': False,               # just leave as is
+            'center_psf': True,                    # True for 1p
+            'only_init': True,                     # set it to True to run CNMF-E
+            'method_deconvolution': 'oasis'}       # could use 'cvxpy' alternatively
 
         if not len(dset):
             dset = (self.schema.TwoPhoton() & self.dataset_key)
@@ -244,7 +252,7 @@ class CaimanCompute(Suite2pCompute):
         print(f"Number accepted: {len(cnmfe_model.estimates.idx_components)}")
         print(f"Number rejected: {len(cnmfe_model.estimates.idx_components_bad)}")
         cluster.terminate()
-        from ..stacks import compute_projections
+        from ..stacks import compute_projections,get_roi_pixels,get_roi_contour
         mean_proj,std_proj,max_proj,corr_proj = compute_projections(images)
         print('Projections computed.')
         import caiman
@@ -330,19 +338,8 @@ class CaimanCompute(Suite2pCompute):
                     os.unlink(f)
         else:
             print(f'[{self.name} job] Kept the temporary folder {temporary_folder}.')
-
-def get_roi_contour(roi_mask,percentile_threshold = 80):
-    from skimage.measure import find_contours
-    # find_contours?
-    level = np.percentile(roi_mask[roi_mask>0], percentile_threshold)
-    C = find_contours(roi_mask,level = level) # there should be only one contour, this takes the first one
-    return C[0]
-    
-def get_roi_pixels(roi_mask):
-    ii = np.ravel_multi_index(np.where(roi_mask!=0), roi_mask.shape)
-    return ii, np.take(roi_mask,ii)
-
-
+        CellSegmentationMetrics().populate(planekey, display_progress = True)
+        
 def setup_cluster(n_cpus,cluster = None):
     from caiman import stop_server, cluster
     #%% start a cluster for parallel processing (if a cluster already exists it will be closed and a new session will be opened)

{labdata-0.1.1 → labdata-0.1.2}/labdata/dashboard/decisiontask.py

@@ -46,7 +46,7 @@ axis = alt.Axis(domain=True, domainColor='black', domainCap='butt',
 
 def compute_psychometric(trialset_rows):
     """Proportion of rightward responses per condition per trialset with Wilson CI.
-    response_values convention: left=1, no response=0, right=-1.
+    response_values convention: side1=1, no response=0, side2=-1.
     """
     from scipy.stats import binomtest
     rows = []
@@ -73,44 +73,58 @@ def compute_psychometric(trialset_rows):
                              trialset_description=desc))
     return pd.DataFrame(rows)
 
-
 def decisionmaking_tab(schema = None):
     @st.cache_data
-    def get_subjects_last_ran():
+    def get_max_date():
+        try:
+            return pd.to_datetime(
+                (schema.DecisionTask.TrialSet() * schema.Session()).fetch('session_datetime')
+            ).max().date()
+        except:
+            return None
+
+    @st.cache_data
+    def get_subjects_last_ran(end_date):
+        date_str = end_date.strftime('%Y-%m-%d')
         subs = schema.Subject & schema.DecisionTask
         try:
-            max_date = (subs).aggr(schema.Session,ss='max(session_datetime)').fetch('ss').max().strftime('%Y-%m-%d')
+            tt = (schema.DecisionTask() * schema.Session() &
+                  f'session_datetime > (DATE("{date_str}") - INTERVAL 3 MONTH)' &
+                  f'DATE(session_datetime) <= "{date_str}"')
+            subjects = (schema.Subject() & tt).fetch(format='frame').reset_index()
+            subjects = np.unique(subjects.subject_name.values)
         except:
-            print('There are no sessions to max.')
-            return [],[],None
-        tt = (schema.DecisionTask()*schema.Session &
-              f'session_datetime > (DATE("{max_date}") - INTERVAL 3 MONTH)')
-        subjects = (schema.Subject() & tt).fetch(format = 'frame').reset_index()
-        subjects = np.unique(subjects.subject_name.values)
-        keys = [dict(subject_name = s) for s in subjects]
-        return subs.fetch('subject_name'), keys, max_date
+            print('There are no sessions.')
+            return [], []
+        keys = [dict(subject_name=s) for s in subjects]
+        return subs.fetch('subject_name'), keys
 
     @st.cache_data
-    def get_summary_data(selected_indices,max_date):
-        keys = [dict(subject_name = i) for i in selected_indices]
-        max_date = schema.Subject.aggr(schema.Session & keys,ss='max(session_datetime)').fetch('ss').max().strftime('%Y-%m-%d')
-        ses = (schema.Session()*schema.DecisionTask.TrialSet() & keys & f'session_datetime > (DATE("{max_date}") - INTERVAL 3 MONTH)').proj().fetch(as_dict = True)
-    
-        from datetime import timedelta
+    def get_summary_data(selected_indices, end_date):
+        keys = [dict(subject_name=i) for i in selected_indices]
+        date_str = end_date.strftime('%Y-%m-%d')
+        date_filter = (f'session_datetime > (DATE("{date_str}") - INTERVAL 3 MONTH) '
+                       f'AND DATE(session_datetime) <= "{date_str}"')
         summarysource = pd.DataFrame((schema.Session*schema.DecisionTask.TrialSet() & keys & 'trialset_description NOT LIKE "%opto%"'
-                                    & f'session_datetime > (DATE("{max_date}") - INTERVAL 3 MONTH)').fetch())
+                                    & date_filter).fetch())
         if not len(summarysource):
             return []
-        days = np.array([(summarysource.session_datetime.max() - timedelta(days=d)).date() for d in range(40,-2,-1)])
         summarysource['date'] = summarysource.session_datetime.map(
                     lambda x: x.strftime("%y-%m-%d"))
         return summarysource
 
-    if st.button('Refresh', icon=':material/refresh:'):
-        get_subjects_last_ran.clear()
-        get_summary_data.clear()
+    ctrl_col, date_col, _ = st.columns([1, 2, 6])
+    with ctrl_col:
+        if st.button('Refresh', icon=':material/refresh:'):
+            get_max_date.clear()
+            get_subjects_last_ran.clear()
+            get_summary_data.clear()
+    with date_col:
+        end_date = st.date_input('End date', value=get_max_date(), label_visibility='collapsed')
 
-    all_subjects,subjects,max_date = get_subjects_last_ran()
+    if end_date is None:
+        return
+    all_subjects, subjects = get_subjects_last_ran(end_date)
     if not len(subjects):
         st.write('There are no sessions.')
         return
@@ -118,10 +132,12 @@ def decisionmaking_tab(schema = None):
     selected_indices = st.multiselect('Select subjects:',all_subjects,default = [s['subject_name'] for s in subjects])
     if len(selected_indices):
         # cache data to run faster
-        summarysource = get_summary_data(selected_indices,max_date)
+        summarysource = get_summary_data(selected_indices, end_date)
         if not len(summarysource):
             st.write('Could not find sessions for the specified dates..')
             return
+
+        # Build heatmap chart
         point_selector = alt.selection_point(on="click")
         chart = alt.Chart(summarysource).mark_rect().encode(
             x=alt.X('session_datetime:T',timeUnit = 'yearmonthdate',
@@ -130,9 +146,13 @@ def decisionmaking_tab(schema = None):
                     color=alt.Color('performance_easy:Q',
                     scale=alt.Scale(scheme='spectral', reverse=True, domain=[0.5,1], clamp=True)).title('Perf easy'),
                     tooltip = ['subject_name:N','session_name:N','session_datetime:T','performance_easy:Q','performance:Q','n_trials:Q']
+        ).properties(title ='Behavioral performance').add_params(point_selector)
+
+        # Heatmap (session selector) and psychometric curve side-by-side
+        col_map, col_psych = st.columns([3, 2])
 
-        ).properties(width=1000, title ='Behavioral performance').add_params(point_selector)
-        event_data = st.altair_chart(chart.interactive(), on_select='rerun')
+        with col_map:
+            event_data = st.altair_chart(chart.interactive(), on_select='rerun', width='stretch')
 
         if len(event_data["selection"]["param_1"]):
             subject_name = event_data["selection"]["param_1"][0]["subject_name"]
@@ -148,7 +168,6 @@ def decisionmaking_tab(schema = None):
                         st.session_state['decision_last_clicked_session'] = new_session
                 except Exception:
                     pass
-            st.write(f'### _{subject_name}_')
 
             @st.cache_resource
             def get_subject_data(subject_name):
@@ -164,10 +183,6 @@ def decisionmaking_tab(schema = None):
                         & dict(subject_name=subject_name, session_name=session_name)
                         ).fetch(*cols, as_dict=True)
 
-            if st.button('Refresh sessions', icon=':material/refresh:'):
-                get_subject_data.clear()
-                get_session_trials.clear()
-
             trialset, weights, watering = get_subject_data(subject_name)
             sessions = sorted(trialset['session_datetime'].unique())
 
@@ -182,7 +197,7 @@ def decisionmaking_tab(schema = None):
                 except Exception:
                     pass
 
-            # resolve selected session before columns so selected_dt is available everywhere
+            # resolve selected session before rendering so selected_dt is available everywhere
             session_names = trialset[['session_datetime','session_name']].drop_duplicates('session_name').sort_values('session_datetime')
             snames = list(session_names['session_name'])
             clicked_session = st.session_state.pop('decision_selected_session', None)
@@ -196,46 +211,19 @@ def decisionmaking_tab(schema = None):
             selected_session = snames[session_idx]
             selected_dt = session_names.loc[session_names['session_name'] == selected_session, 'session_datetime'].iloc[0]
 
-            col_summary, _, col_psych = st.columns([2, 0.15, 1])
-
-            with col_summary:
-                black, orange = '#000000','#ff7f0e'
-                perf_val = trialset.dropna(subset=['performance_easy','performance']).sort_values('session_datetime').drop_duplicates('session_datetime').iloc[[-1]]
-                x_enc = alt.X('session_datetime:T', axis=axis).title('Session date')
-                perf_chart = (
-                    alt.Chart(trialset).mark_line(color=black).encode(x=x_enc, y=alt.Y('performance_easy:Q', axis=axis).title('Performance')).properties(width=1000) +
-                    alt.Chart(trialset).mark_point(color=black).encode(x=x_enc, y=alt.Y('performance_easy:Q'), tooltip=['session_name:N','session_datetime:T','performance_easy:Q','performance:Q']) +
-                    alt.Chart(perf_val).mark_text(align='left', dx=6, dy=100, color=black).encode(x=x_enc, y=alt.Y('performance_easy:Q'), text=alt.value('easy')) +
-                    alt.Chart(trialset).mark_line(color=orange).encode(x=x_enc, y=alt.Y('performance:Q', axis=axis)) +
-                    alt.Chart(trialset).mark_point(color=orange).encode(x=x_enc, y=alt.Y('performance:Q'),
-                                                                        tooltip=['session_name:N','session_datetime:T','performance_easy:Q','performance:Q']) +
-                    alt.Chart(perf_val).mark_text(align='left', dx=6, dy=100, color=orange).encode(x=x_enc, y=alt.Y('performance:Q'), text=alt.value('all trials'))
-                )
-                ruler_df = pd.DataFrame({'t': [selected_dt]})
-                def make_ruler(x_field):
-                    return alt.Chart(ruler_df).mark_rule(
-                        color='grey', strokeDash=[10, 10], strokeWidth=1.5
-                    ).encode(x=alt.X(f't:T'))
-
-                chartw = line_point_plot(weights,
-                                x=alt.X('weighing_datetime:T',axis=axis).title(''),
-                                y=alt.Y('weight:Q',axis=axis).title('Weight (g)'),
-                                tooltip=['weighing_datetime', 'weight'],
-                                color = colors[0])
-                charth2o = line_point_plot(watering,
-                                x=alt.X('watering_datetime:T',axis=axis).title('Session date'),
-                                y=alt.Y('water_volume:Q',axis=axis).title('Water volume (mL)'),
-                                tooltip=['watering_datetime', 'water_volume'],
-                                color = colors[2])
-                linked = alt.vconcat(
-                    perf_chart + make_ruler('session_datetime'),
-                    chartw    + make_ruler('weighing_datetime'),
-                    charth2o  + make_ruler('watering_datetime'),
-                ).resolve_scale(x='shared')
-                st.altair_chart(linked.interactive(), width="stretch")
+            with col_map:
+                if st.button('Refresh sessions', icon=':material/refresh:'):
+                    get_subject_data.clear()
+                    get_session_trials.clear()
 
+            # Psychometric curve and reaction time in right column
             with col_psych:
-
+                name_col, slider_col = st.columns([1, 3])
+                name_col.write(f'### _{subject_name}_')
+                if len(snames) > 1:
+                    with slider_col:
+                        st.slider('Session', min_value=0, max_value=len(snames)-1,
+                                  key='decision_session_slider')
                 trial_rows = get_session_trials(subject_name, selected_session)
                 psych_df = compute_psychometric(trial_rows)
                 if not psych_df.empty:
@@ -258,8 +246,8 @@ def decisionmaking_tab(schema = None):
                     )
                     lines = base.mark_line().encode(y=alt.Y('proportion:Q'))
                     st.altair_chart((lines + errorbars + pts).properties(
-                        width=400, height=500, title=f'{pd.Timestamp(selected_dt).strftime("%Y-%m-%d %H:%M")}'
-                    ).interactive(), width="stretch")
+                        width=400, height=400, title=f'{pd.Timestamp(selected_dt).strftime("%Y-%m-%d %H:%M")}'
+                    ).interactive(), width='stretch')
 
                     # reaction time per condition
                     rt_rows = []
@@ -298,15 +286,11 @@ def decisionmaking_tab(schema = None):
                         )
                         rt_lines = rt_base.mark_line().encode(y=alt.Y('median_rt:Q'))
                         st.altair_chart((rt_lines + rt_err + rt_pts).properties(
-                            width=400, height=300, title='Reaction time per condition'
-                        ).interactive(), width="stretch")
+                            width=400, height=400, title='Reaction time per condition'
+                        ).interactive(), width='stretch')
                 else:
                     st.error('No data available for this session.')
 
-                if len(snames) > 1:
-                    st.slider('Session', min_value=0, max_value=len(snames)-1,
-                              key='decision_session_slider')
-
                 @st.cache_data
                 def get_session_notes(subject_name, session_name):
                     rows = (schema.Dataset * schema.Note
@@ -320,11 +304,44 @@ def decisionmaking_tab(schema = None):
                         st.caption(f"*{n['notetaker']} — {pd.Timestamp(n['note_datetime']).strftime('%Y-%m-%d %H:%M')}*")
                         st.write(n['notes'])
 
-def line_point_plot(data, x, y, tooltip, color):
+            # Performance and watering charts under the colormap in the left column (80% width)
+            with col_map:
+                perf_col, _ = st.columns([4, 1])
+                black, orange = '#000000','#ff7f0e'
+                perf_val = trialset.dropna(subset=['performance_easy','performance']).sort_values('session_datetime').drop_duplicates('session_datetime').iloc[[-1]]
+                x_enc = alt.X('session_datetime:T', axis=axis).title('Session date')
+                perf_chart = (
+                    alt.Chart(trialset).mark_line(color=black).encode(x=x_enc, y=alt.Y('performance_easy:Q', axis=axis).title('Performance')).properties(width=400, height=150) +
+                    alt.Chart(trialset).mark_point(color=black).encode(x=x_enc, y=alt.Y('performance_easy:Q'), tooltip=['session_name:N','session_datetime:T','performance_easy:Q','performance:Q']) +
+                    alt.Chart(perf_val).mark_text(align='left', dx=6, dy=100, color=black).encode(x=x_enc, y=alt.Y('performance_easy:Q'), text=alt.value('easy')) +
+                    alt.Chart(trialset).mark_line(color=orange).encode(x=x_enc, y=alt.Y('performance:Q', axis=axis)) +
+                    alt.Chart(trialset).mark_point(color=orange).encode(x=x_enc, y=alt.Y('performance:Q'),
+                                                                        tooltip=['session_name:N','session_datetime:T','performance_easy:Q','performance:Q']) +
+                    alt.Chart(perf_val).mark_text(align='left', dx=6, dy=100, color=orange).encode(x=x_enc, y=alt.Y('performance:Q'), text=alt.value('all trials'))
+                )
+                chartw = line_point_plot(weights,
+                                x=alt.X('weighing_datetime:T',axis=axis).title(''),
+                                y=alt.Y('weight:Q',axis=axis).title('Weight (g)'),
+                                tooltip=['weighing_datetime', 'weight'],
+                                color=colors[0], height=150)
+                charth2o = line_point_plot(watering,
+                                x=alt.X('watering_datetime:T',axis=axis).title('Session date'),
+                                y=alt.Y('water_volume:Q',axis=axis).title('Water volume (mL)'),
+                                tooltip=['watering_datetime', 'water_volume'],
+                                color=colors[2], height=150)
+                linked = alt.vconcat(
+                    perf_chart,
+                    chartw,
+                    charth2o,
+                ).resolve_scale(x='shared')
+                with perf_col:
+                    st.altair_chart(linked.interactive(), width='stretch')
+
+def line_point_plot(data, x, y, tooltip, color, height=150):
     if data is None or not len(data):
         return alt.Chart(pd.DataFrame()).mark_point()
     scatter = alt.Chart(data).mark_point(color=color).encode(
                 x=x, y=y, tooltip=tooltip)
     line = alt.Chart(data).mark_line(color=color).encode(
-                x=x, y=y).properties(width=1000)
+                x=x, y=y).properties(width=400, height=height)
     return scatter + line