labdata 0.0.24__tar.gz → 0.0.25__tar.gz

{labdata-0.0.24 → labdata-0.0.25}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: labdata
-Version: 0.0.24
+Version: 0.0.25
 Summary: Package to manage data in experimental neuroscience labs
 Author-email: Joao Couto <jpcouto@gmail.com>
 Project-URL: Homepage, https://github.com/jcouto/labdata
@@ -23,7 +23,7 @@ Requires-Dist: dash; extra == "dashboard"
 Requires-Dist: dash_bootstrap_components; extra == "dashboard"
 Requires-Dist: dash-extensions; extra == "dashboard"
 Requires-Dist: plotly; extra == "dashboard"
-Requires-Dist: streamlit>1.34; extra == "dashboard"
+Requires-Dist: streamlit>1.54; extra == "dashboard"
 Requires-Dist: streamlit-bokeh; extra == "dashboard"
 Requires-Dist: scipy; extra == "dashboard"
 Requires-Dist: scikit-image; extra == "dashboard"

{labdata-0.0.24 → labdata-0.0.25}/labdata/__init__.py

@@ -1,4 +1,4 @@
-VERSION = "0.0.24"
+VERSION = "0.0.25"
 
 from .utils import *
 from .copy import copy_to_upload_server

{labdata-0.0.24 → labdata-0.0.25}/labdata/dashboard/compute.py

@@ -44,7 +44,7 @@ def _jobs_panel(table, status_field, target_field, cols, delete_key):
             x='Jobs:Q',
             color=alt.Color('color:N').scale(None)
         ).configure_axis(labelLimit=3000))
-        st.altair_chart(ch, use_container_width=True)
+        st.altair_chart(ch, width="stretch")
 
     # filtered table
     selected_states = st.multiselect('Select status:', STATES, default=STATES[1:], key=f'{delete_key}_filter')
@@ -58,7 +58,7 @@ def _jobs_panel(table, status_field, target_field, cols, delete_key):
         display_df,
         column_config={'Select': st.column_config.CheckboxColumn(required=True)},
         disabled=[c for c in display_df.columns if c != 'Select'],
-        use_container_width=True,
+        width="stretch",
         key=f'{delete_key}_editor',
     )
     selected_jobs = edited[edited['Select'] == True].index.tolist()

{labdata-0.0.24 → labdata-0.0.25}/labdata/dashboard/decisiontask.py

@@ -106,11 +106,15 @@ def decisionmaking_tab(schema = None):
                     lambda x: x.strftime("%y-%m-%d"))
         return summarysource
 
+    if st.button('Refresh', icon=':material/refresh:'):
+        get_subjects_last_ran.clear()
+        get_summary_data.clear()
+
     all_subjects,subjects,max_date = get_subjects_last_ran()
     if not len(subjects):
         st.write('There are no sessions.')
         return
-    
+
     selected_indices = st.multiselect('Select subjects:',all_subjects,default = [s['subject_name'] for s in subjects])
     if len(selected_indices):
         # cache data to run faster
@@ -124,7 +128,7 @@ def decisionmaking_tab(schema = None):
                     scale=alt.Scale(nice={'interval': 'day', 'step': 1})).axis(labelAngle=45,format='%Y-%m-%d').title('Date'),  
             y=alt.Y('subject_name:O').title('Subject name'),
                     color=alt.Color('performance_easy:Q',
-                    scale=alt.Scale(scheme='spectral', reverse=True,domain = [0.5,1])).title('Perf easy'),
+                    scale=alt.Scale(scheme='spectral', reverse=True, domain=[0.5,1], clamp=True)).title('Perf easy'),
                     tooltip = ['subject_name','session_name','session_datetime','performance_easy','performance','n_trials']
 
         ).properties(width=1000, title ='Behavioral performance').add_params(point_selector)
@@ -160,6 +164,10 @@ def decisionmaking_tab(schema = None):
                         & dict(subject_name=subject_name, session_name=session_name)
                         ).fetch(*cols, as_dict=True)
 
+            if st.button('Refresh sessions', icon=':material/refresh:'):
+                get_subject_data.clear()
+                get_session_trials.clear()
+
             trialset, weights, watering = get_subject_data(subject_name)
             sessions = sorted(trialset['session_datetime'].unique())
 
@@ -192,7 +200,7 @@ def decisionmaking_tab(schema = None):
 
             with col_summary:
                 black, orange = '#000000','#ff7f0e'
-                perf_val = trialset.dropna(subset=['performance_easy','performance']).sort_values('session_datetime').drop_duplicates('session_datetime').iloc[[-20]]
+                perf_val = trialset.dropna(subset=['performance_easy','performance']).sort_values('session_datetime').drop_duplicates('session_datetime').iloc[[-1]]
                 x_enc = alt.X('session_datetime:T', axis=axis).title('Session date')
                 perf_chart = (
                     alt.Chart(trialset).mark_line(color=black).encode(x=x_enc, y=alt.Y('performance_easy:Q', axis=axis).title('Performance')).properties(width=1000) +
@@ -224,7 +232,7 @@ def decisionmaking_tab(schema = None):
                     chartw    + make_ruler('weighing_datetime'),
                     charth2o  + make_ruler('watering_datetime'),
                 ).resolve_scale(x='shared')
-                st.altair_chart(linked.interactive(), use_container_width=True)
+                st.altair_chart(linked.interactive(), width="stretch")
 
             with col_psych:
 
@@ -251,7 +259,7 @@ def decisionmaking_tab(schema = None):
                     lines = base.mark_line().encode(y=alt.Y('proportion:Q'))
                     st.altair_chart((lines + errorbars + pts).properties(
                         width=400, height=500, title=f'{pd.Timestamp(selected_dt).strftime("%Y-%m-%d %H:%M")}'
-                    ).interactive(), use_container_width=True)
+                    ).interactive(), width="stretch")
 
                     # reaction time per condition
                     rt_rows = []
@@ -291,9 +299,9 @@ def decisionmaking_tab(schema = None):
                         rt_lines = rt_base.mark_line().encode(y=alt.Y('median_rt:Q'))
                         st.altair_chart((rt_lines + rt_err + rt_pts).properties(
                             width=400, height=300, title='Reaction time per condition'
-                        ).interactive(), use_container_width=True)
+                        ).interactive(), width="stretch")
                 else:
-                    st.write('No response data available for this session.')
+                    st.error('No data available for this session.')
 
                 if len(snames) > 1:
                     st.slider('Session', min_value=0, max_value=len(snames)-1,

labdata-0.0.25/labdata/dashboard/index.py

@@ -0,0 +1,225 @@
+import streamlit as st
+import pandas as pd
+import numpy as np
+from functools import partial
+
+import sys
+
+def intro_tab(user_name = None, schema = None):
+    @st.cache_data
+    def get_subjects():
+        if user_name is not None:
+            df = pd.DataFrame((schema.Subject() & f'user_name = "{user_name}"').fetch())
+        else:
+            df = pd.DataFrame(schema.Subject().fetch())
+        return df.sort_values("subject_name").reset_index(drop=True)
+
+    @st.cache_data
+    def get_sessions(subject_names):
+        dfs = [pd.DataFrame((schema.Session() * schema.Dataset() &
+                              f'subject_name = "{s}"').fetch()) for s in subject_names]
+        dfs = [d for d in dfs if len(d)]
+        if not dfs:
+            return None
+        return pd.concat(dfs).sort_values("session_datetime").reset_index(drop=True)
+
+    @st.cache_data
+    def get_files(dataset_keys_frozen):
+        keys = [dict(k) for k in dataset_keys_frozen]
+        rows = (schema.Dataset.DataFiles * schema.File & keys).fetch(
+            'subject_name', 'session_name', 'dataset_name',
+            'file_path', 'file_size', 'file_datetime', 'storage', as_dict=True)
+        if not rows:
+            return pd.DataFrame()
+        df = pd.DataFrame(rows)
+        df['size'] = df['file_size'].apply(
+            lambda b: f'{b/1e9:.2f} GB' if b >= 1e9 else f'{b/1e6:.1f} MB' if b >= 1e6 else f'{b/1e3:.0f} KB')
+        return df.drop(columns='file_size')
+
+    @st.cache_data
+    def get_users():
+        return list(schema.LabMember.fetch('user_name'))
+
+    @st.cache_data
+    def get_strains():
+        return list(schema.Strain().fetch('strain_name'))
+
+    list_tab, add_tab, edit_tab = st.tabs(['Subjects/Sessions', 'New', 'Update'])
+
+    with list_tab:
+        subjects = get_subjects()
+        event_subjects = st.dataframe(
+            subjects,
+            hide_index=True,
+            width="stretch",
+            on_select='rerun',
+            selection_mode='multi-row',
+            key='subjects_table',
+        )
+        sel_subj_idx = (event_subjects.selection or {}).get('rows', [])
+        selected_names = list(subjects.iloc[sel_subj_idx]['subject_name']) if sel_subj_idx else []
+        if selected_names:
+            st.session_state['subjects_selected'] = selected_names
+
+        sessions = get_sessions(selected_names) if selected_names else None
+        if sessions is None:
+            st.write('No subjects selected.')
+        else:
+            dtype_vals = [s if s is not None else 'None' for s in sessions.dataset_type.values]
+            uniqueds = np.unique(dtype_vals)
+            metric_cols = st.columns(max(1, len(uniqueds)))
+            for col, d in zip(metric_cols, uniqueds):
+                col.metric(d, sum(1 for v in dtype_vals if v == d))
+
+            # color palette for dataset types (light backgrounds, readable text)
+            type_palette = ['#d0e8ff', '#ffd6d6', '#d4f5d4', '#fff0cc',
+                            '#e8d4f5', '#ffd9b3', '#d4f0f5', '#f5d4e8',
+                            '#e8f5d4', '#f5e8d4']
+            dtype_list = [s if s is not None else 'None' for s in sessions['dataset_type'].values]
+            unique_types = list(dict.fromkeys(dtype_list))  # preserve order, deduplicate
+            type_color = {t: type_palette[i % len(type_palette)] for i, t in enumerate(unique_types)}
+
+            def _color_rows(row):
+                dt = row.get('dataset_type')
+                dt = dt if dt is not None else 'None'
+                bg = type_color.get(dt, '')
+                return [f'background-color: {bg}'] * len(row)
+
+            sess_styled = sessions.style.apply(_color_rows, axis=1)
+            event_sessions = st.dataframe(
+                sess_styled,
+                hide_index=True,
+                width="stretch",
+                on_select='rerun',
+                selection_mode='multi-row',
+                key='sessions_table',
+            )
+            sel_indices = (event_sessions.selection or {}).get('rows', [])
+            sel_datasets = sessions.iloc[sel_indices] if sel_indices else pd.DataFrame()
+            if len(sel_datasets):
+                dataset_keys = sel_datasets[['subject_name', 'session_name', 'dataset_name']].to_dict('records')
+                file_stats = get_files(tuple(frozenset(d.items()) for d in dataset_keys))
+                if len(file_stats):
+                    st.write('### Files')
+                    ds_order = list(dict.fromkeys(file_stats['dataset_name'].values))
+                    even_color, odd_color = '#dceeff', '#fff3dc'
+                    ds_color = {ds: (even_color if i % 2 == 0 else odd_color)
+                                for i, ds in enumerate(ds_order)}
+
+                    def _color_file_rows(row):
+                        bg = ds_color.get(row['dataset_name'], '')
+                        return [f'background-color: {bg}'] * len(row)
+
+                    file_styled = file_stats.style.apply(_color_file_rows, axis=1)
+                    event_files = st.dataframe(
+                        file_styled,
+                        hide_index=True,
+                        width="stretch",
+                        on_select='rerun',
+                        selection_mode='multi-row',
+                        key='files_table',
+                    )
+                    sel_file_idx = (event_files.selection or {}).get('rows', [])
+                    if sel_file_idx:
+                        sel_files = file_stats.iloc[sel_file_idx]
+                        st.caption(f'{len(sel_file_idx)} file(s) selected')
+                        if st.button('Download files', key='download_files_btn'):
+                            keys = sel_files[['file_path', 'storage']].to_dict('records')
+                            with st.spinner(f'Downloading {len(sel_file_idx)} file(s)…'):
+                                local = (schema.File & keys).get()
+                            if local:
+                                st.success(f'Downloaded {len(local)} file(s).')
+                                for p in local:
+                                    st.code(str(p))
+                else:
+                    st.write('No files found for the selected datasets.')
+
+    with add_tab:
+        users = get_users()
+        available_strains = get_strains()
+        project_users = list((schema.Project.User() & f'project_name = "{schema.schema_project}"').fetch('user_name'))
+        userindex = users.index(project_users[0]) if project_users and project_users[0] in users else None
+        with st.form('add_subject'):
+            insert_dict = dict()
+            insert_dict['user_name'] = st.selectbox('User Name', users, index=userindex)
+            insert_dict['subject_name'] = st.text_input('Subject ID', value=None)
+            insert_dict['subject_dob'] = st.date_input('Date of Birth')
+            sex = st.selectbox('Sex', ['M', 'F', 'Unknown'])
+            insert_dict['subject_sex'] = 'U' if sex == 'Unknown' else sex
+            insert_dict['strain_name'] = st.selectbox('Strain', available_strains)
+            if st.form_submit_button('Add Subject', type='primary'):
+                schema.Subject().insert1(insert_dict)
+                get_subjects.clear()
+                st.success(f'Added subject {insert_dict["subject_name"]}.')
+                st.rerun()
+
+    with edit_tab:
+        users = get_users()
+        available_strains = get_strains()
+        selected_names = st.session_state.get('subjects_selected', [])
+        if not selected_names:
+            st.info('Select a subject in the Subjects/Sessions tab first.')
+        else:
+            selected_subject = selected_names[0]
+            st.write(f'### Edit subject: {selected_subject}')
+            with st.form('edit_subject'):
+                edit_dict = (schema.Subject & dict(subject_name=selected_subject)).fetch1()
+                edit_dict['subject_dob'] = st.date_input('Date of Birth', value=edit_dict['subject_dob'])
+                edit_dict['subject_sex'] = st.selectbox('Sex', ['M', 'F', 'U'],
+                                                         index=['M', 'F', 'U'].index(edit_dict['subject_sex']))
+                edit_dict['strain_name'] = st.selectbox('Strain', available_strains,
+                                                         index=available_strains.index(edit_dict['strain_name']))
+                if st.form_submit_button('Save changes', type='primary'):
+                    schema.Subject().update1(edit_dict)
+                    get_subjects.clear()
+                    st.success(f'Updated {selected_subject}.')
+                    st.rerun()
+st.set_page_config(
+    page_title="labdata dashboard",
+    page_icon="🧪",
+    layout="wide",
+      initial_sidebar_state="auto")
+
+
+from procedures import procedures_tab
+from notes import notes_tab
+from sorting import sorting_tab
+from segmentation import segmentation_tab
+from video import video_tab
+from decisiontask import decisionmaking_tab
+from compute import compute_tab
+
+@st.cache_resource
+def load_schema(project):
+    # load project
+    from labdata import load_project_schema
+    return load_project_schema(project)
+
+project = None
+for k in sys.argv:
+    if 'project=' in k:
+        project = k.replace('project=','')
+with st.spinner(f'Connecting to {"project " + project if project else "database"}…'):
+    schema = load_schema(project)
+
+page_names_to_funcs = {
+    "**Subjects**": partial(intro_tab, schema = schema),
+    "**Procedures**": partial(procedures_tab, schema = schema),
+    "**Notes**": partial(notes_tab, schema = schema),
+    "**Spike sorting**": partial(sorting_tab, schema = schema),
+    "**Cell Segmentation**": partial(segmentation_tab, schema = schema),
+    "**Video**": partial(video_tab, schema = schema),
+    "**Decision-making behavior**": partial(decisionmaking_tab, schema = schema),
+    "**Compute/Upload**": partial(compute_tab, schema = schema),
+}
+
+from labdata import plugins
+for p in plugins.keys():
+    if hasattr(plugins[p],'dashboard_function'):
+        page_names_to_funcs[plugins[p].dashboard_name] = partial(plugins[p].dashboard_function, schema = schema)
+
+tab_name = st.sidebar.radio(
+    "``labdata dashboard``",
+    page_names_to_funcs.keys())
+
+page_names_to_funcs[tab_name]()

labdata-0.0.25/labdata/dashboard/notes.py

@@ -0,0 +1,184 @@
+import pandas as pd
+import streamlit as st
+from datetime import datetime, date, time as dtime
+
+
+def notes_tab(schema=None):
+
+    @st.cache_data
+    def get_users():
+        return list(schema.LabMember.fetch('user_name'))
+
+    @st.cache_data
+    def get_subjects():
+        return list(schema.Subject.fetch('subject_name'))
+
+    @st.cache_data
+    def get_sessions(subject_name):
+        return list((schema.Session & dict(subject_name=subject_name)).fetch('session_name'))
+
+    @st.cache_data
+    def get_datasets(subject_name, session_name):
+        return list((schema.Dataset & dict(subject_name=subject_name,
+                                           session_name=session_name)).fetch('dataset_name'))
+
+    @st.cache_data
+    def get_note_counts():
+        rows = (schema.Subject.aggr(
+            schema.Dataset * schema.Note,
+            note_count='count(*)'
+        )).fetch(as_dict=True)
+        df = pd.DataFrame(rows)
+        if len(df):
+            return df[df['note_count'] > 0].sort_values('note_count', ascending=False).reset_index(drop=True)
+        return []
+
+    @st.cache_data
+    def get_subject_notes(subject_name):
+        query = schema.Dataset * schema.Session * schema.Note & dict(subject_name=subject_name)
+        rows = query.fetch(as_dict=True)
+        if not rows:
+            return pd.DataFrame()
+        df = pd.DataFrame(rows)
+        cols = ['subject_name', 'session_name', 'dataset_name', 'session_datetime',
+                'notes', 'notetaker', 'note_datetime']
+        cols = [c for c in cols if c in df.columns]
+        return df[cols].sort_values('note_datetime', ascending=False).reset_index(drop=True)
+
+    users = get_users()
+    subjects = get_subjects()
+
+    browse_tab, update_tab, add_tab = st.tabs(['Browse', 'Update', 'Create'])
+
+    def _render_notes_table(notes_df, editor_key):
+        display_df = notes_df.copy()
+        display_df.insert(0, 'Select', False)
+        edited = st.data_editor(
+            display_df,
+            column_config={'Select': st.column_config.CheckboxColumn(required=True)},
+            disabled=[c for c in display_df.columns if c != 'Select'],
+            width="stretch",
+            key=editor_key,
+        )
+        selected_rows = edited[edited['Select'] == True]
+        if len(selected_rows) > 1:
+            st.warning('Multiple notes selected — selecting the first one.')
+        if len(selected_rows):
+            st.session_state['selected_note'] = selected_rows.iloc[0].to_dict()
+
+    with browse_tab:
+        import altair as alt
+        counts_df = get_note_counts()
+        subject_filter = None
+        if len(counts_df):
+            sel = alt.selection_point(fields=['subject_name'], name='subject_sel')
+            chart = (alt.Chart(counts_df).mark_bar().encode(
+                x=alt.X('subject_name:N', sort='ascending', title='Subject'),
+                y=alt.Y('note_count:Q', title='# notes'),
+                color=alt.condition(sel, alt.value('black'), alt.value('#aaaaaa')),
+                tooltip=['subject_name:N', 'note_count:Q'],
+            ).add_params(sel).properties(height=200))
+            event = st.altair_chart(chart, width="stretch", on_select='rerun',
+                                    key='notes_count_chart')
+            if st.session_state.pop('notes_clear_pending', False):
+                st.session_state.pop('notes_selected_subject', None)
+            else:
+                pts = (event.selection or {}).get('subject_sel', [])
+                if pts and pts[0].get('subject_name'):
+                    st.session_state['notes_selected_subject'] = pts[0]['subject_name']
+            subject_filter = st.session_state.get('notes_selected_subject')
+            if subject_filter:
+                st.write(f'**Selected:** {subject_filter}')
+                if st.button('Clear selection', key='notes_clear_subject'):
+                    st.session_state['notes_clear_pending'] = True
+                    st.rerun()
+        else:
+            st.write('No notes in the database.')
+
+        if subject_filter:
+            notes_df = get_subject_notes(subject_filter)
+            if len(notes_df):
+                if len(notes_df) > 1:
+                    idx = st.slider('Note', min_value=1, max_value=len(notes_df), value=1,
+                                    format='%d', key='notes_slider',
+                                    help='Cycle through notes (sorted by date, newest first)')
+                    row = notes_df.iloc[idx - 1]
+                    st.write(f"**{row.get('session_datetime', '')}** — {row.get('dataset_name', '')} — *{row.get('notetaker', '')}*")
+                    st.write(row.get('notes', ''))
+                    st.session_state['selected_note'] = row.to_dict()
+                    with st.expander('Show all notes as table'):
+                        _render_notes_table(notes_df, 'notes_browse_editor')
+                else:
+                    _render_notes_table(notes_df, 'notes_browse_editor')
+            else:
+                st.write('No notes found for this subject.')
+
+    with add_tab:
+        st.write('### Add a note to a dataset')
+        subject_name = st.selectbox('Subject', subjects, index=None, key='notes_add_subject')
+        session_name = None
+        dataset_name = None
+        if subject_name:
+            sessions = get_sessions(subject_name)
+            session_name = st.selectbox('Session', sessions, index=None, key='notes_add_session')
+        if session_name:
+            datasets = get_datasets(subject_name, session_name)
+            dataset_name = st.selectbox('Dataset', datasets, index=None, key='notes_add_dataset')
+        with st.form('add_note'):
+            project_users = (schema.Project.User() & f'project_name = "{schema.schema_project}"').fetch('user_name')
+            user_index = None
+            if len(project_users) and project_users[0] in users:
+                user_index = users.index(project_users[0])
+            notetaker = st.selectbox('Notetaker', users, index=user_index)
+            note_date = st.date_input('Date', value=date.today())
+            note_time = st.time_input('Time', value=dtime(datetime.now().hour, datetime.now().minute))
+            note_text = st.text_area('Note (max 4000 characters)', max_chars=4000)
+            submitted = st.form_submit_button('Add Note', type='primary')
+            if submitted:
+                if not subject_name:
+                    st.error('Select a subject.')
+                elif not session_name:
+                    st.error('Select a session.')
+                elif not dataset_name:
+                    st.error('Select a dataset.')
+                elif not notetaker:
+                    st.error('Select a notetaker.')
+                elif not note_text.strip():
+                    st.error('Note text cannot be empty.')
+                else:
+                    note_dt = datetime.combine(note_date, note_time)
+                    dataset_key = dict(subject_name=subject_name, session_name=session_name,
+                                       dataset_name=dataset_name)
+                    schema.Note.insert1(dict(notetaker=notetaker, note_datetime=note_dt, notes=note_text))
+                    schema.Dataset.update1({**dataset_key, 'notetaker': notetaker, 'note_datetime': note_dt})
+                    get_note_counts.clear()
+                    get_subject_notes.clear()
+                    st.success(f'Note {dataset_key} added successfully.')
+
+    with update_tab:
+        selected = st.session_state.get('selected_note', {})
+        if not selected:
+            st.info('Select a note in the Browse tab first.')
+        else:
+            notetaker_upd = selected.get('notetaker')
+            note_dt_upd = selected.get('note_datetime')
+            pre_text = selected.get('notes', '')
+            subject_name_upd = selected.get('subject_name')
+            session_name_upd = selected.get('session_name')
+            dataset_name_upd = selected.get('dataset_name')
+            if subject_name_upd and session_name_upd and dataset_name_upd:
+                st.write(f"**Subject:** {subject_name_upd}   **Session:** {session_name_upd}   **Dataset:** {dataset_name_upd}")
+            st.write(f"**Notetaker:** {notetaker_upd}   **Date/time:** {note_dt_upd}")
+            with st.form('update_note'):
+                new_text = st.text_area('Note', value=pre_text, max_chars=4000)
+                submitted_upd = st.form_submit_button('Update Note', type='primary')
+                if submitted_upd:
+                    if not new_text.strip():
+                        st.error('Note text cannot be empty.')
+                    else:
+                        schema.Note.update1(dict(notetaker=notetaker_upd, note_datetime=note_dt_upd,
+                                                 notes=new_text))
+                        get_note_counts.clear()
+                        get_subject_notes.clear()
+                        st.session_state.pop('selected_note', None)
+                        st.success(f'Note {dict(notetaker=notetaker_upd, note_datetime=note_dt_upd)} updated.')